et_number_of_bins=cfg["features"]["binned"], et_embedding_dimension=FLAGS.et_embedding_dimension, with_eeg=cfg["features"]["eeg"], eeg_features_size=eeg.shape[2], use_normalization_layer=True) cnn = TextCNN( sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=embedding_dimension, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, weights=class_weights_for_cross_entropy, rel_pos_embedding_size=FLAGS.rel_pos_embedding_size, rel_pos_cardinality=relative_positions_cardinality, pos_tags_embedding_size=FLAGS.pos_tags_embedding_size, pos_tags_cardinality=pos_tags_cardinality, with_eye_tracking=cfg["features"]["gaze"], et_features_size=et.shape[2], et_number_of_bins=cfg["features"]["binned"], et_embedding_dimension=FLAGS.et_embedding_dimension, with_eeg=cfg["features"]["eeg"], eeg_features_size=eeg.shape[2]) # Define Training procedure global_steps = [ tf.Variable(0, name="global_step_1", trainable=False), tf.Variable(0, name="global_step_2", trainable=False) ]
y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:] print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training # ======================================================= with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None:
# print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # 打印切分的比例 # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=x.shape[1], num_classes=y.shape[1], vocab_size=len(vocab_processor.vocabulary_), # 计算单词的数目 embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split( ","))), # 上面定义的filter_sizes拿过来,"3,4,5"按","分割 num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, # l2正则化项 trainable=not FLAGS.static) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) # 首先将step定义为变量初始化为0 optimizer = tf.train.AdamOptimizer(1e-3) # 定义优化器使用adam优化 grads_and_vars = optimizer.compute_gradients( cnn.loss ) # 将使用卷积神经网络计算出来的损失函数最小化。 该方法会返回list[(gradients,variable)] train_op = optimizer.apply_gradients( grads_and_vars, global_step=global_step) # 使用优化器更新参数,每进行一次参数更新就加一次global step
def train(x_train, y_train, vocab_processor, x_dev, y_dev): # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables print( "SESS PRINT===========================================================" ) print(sess.run(tf.global_variables_initializer())) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) accuracies.append(accuracy) steps.append(step) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
import data_helpers import os import time import tensorflow as tf import datetime with tf.Graph().as_default(): start_time = time.time() session_conf = tf.ConfigProto( allow_soft_placement=CNN.FLAGS.allow_soft_placement, log_device_placement=CNN.FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(filter_sizes=list( map(int, CNN.FLAGS.filter_sizes.split(","))), num_filters=CNN.FLAGS.num_filters, vec_shape=(4, CNN.FLAGS.embedding_size), l2_reg_lambda=CNN.FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.histogram_summary( "{}/grad/hist".format(v.name), g)
print("Vocabulary Size: {:d}".format(len(vocabulary))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=2, vocab_size=len(vocabulary), embedding_size=FLAGS.embedding_dim, filter_sizes=map(int, FLAGS.filter_sizes.split(",")), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-4) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None:
def train_cnn(): """Step 0: load sentences, labels, and training parameters""" dataset = './dataset' x_raw, y_raw, df, labels = data_helper.load_data_and_labels(dataset) parameter_file = "./parameters.json" params = json.loads(open(parameter_file).read()) """Step 1: pad each sentence to the same length and map each word to an id""" max_document_length = max([len(x.split(' ')) for x in x_raw]) logging.info( 'The maximum length of all sentences: {}'.format(max_document_length)) vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) x = np.array(list(vocab_processor.fit_transform(x_raw))) y = np.array(y_raw) """Step 2: split the original dataset into train and test sets""" x_, x_test, y_, y_test = train_test_split(x, y, test_size=0.1, random_state=42) """Step 3: shuffle the train set and split the train set into train and dev sets""" shuffle_indices = np.random.permutation(np.arange(len(y_))) x_shuffled = x_[shuffle_indices] y_shuffled = y_[shuffle_indices] x_train, x_dev, y_train, y_dev = train_test_split(x_shuffled, y_shuffled, test_size=0.1) """Step 4: save the labels into labels.json since predict.py needs it""" with open('./labels.json', 'w') as outfile: json.dump(labels, outfile, indent=4) outfile.close() timestamp = str(int(time.time())) model_folder_name = timestamp if not os.path.exists("trained_model_" + model_folder_name): os.mkdir("trained_model_" + model_folder_name) out_dir = os.path.abspath( os.path.join(os.path.curdir, "trained_model_" + model_folder_name)) os.rename("./labels.json", "./trained_model_" + model_folder_name + "/labels.json") """input category name and save category.json""" cateName = input("카테고리 대분류 명을 입력해 주세요(한글입력) : ") cateJson = {'name': cateName} with open('./category.json', 'w', encoding='utf-8') as cateFile: json.dump(cateJson, cateFile, indent=4, ensure_ascii=False) os.rename("./category.json", "./trained_model_" + model_folder_name + "/category.json") cateFile.close() logging.info('x_train: {}, x_dev: {}, x_test: {}'.format( len(x_train), len(x_dev), len(x_test))) logging.info('y_train: {}, y_dev: {}, y_test: {}'.format( len(y_train), len(y_dev), len(y_test))) """Step 5: build a graph and cnn object""" graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=params['embedding_dim'], filter_sizes=list( map(int, params['filter_sizes'].split(","))), num_filters=params['num_filters'], l2_reg_lambda=params['l2_reg_lambda']) global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables()) # One training step: train the model with one batch def train_step(x_batch, y_batch): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: params['dropout_keep_prob'] } _, step, loss, acc = sess.run( [train_op, global_step, cnn.loss, cnn.accuracy], feed_dict) # One evaluation step: evaluate the model with one batch def dev_step(x_batch, y_batch): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, loss, acc, num_correct = sess.run( [global_step, cnn.loss, cnn.accuracy, cnn.num_correct], feed_dict) return num_correct # Save the word_to_id map since predict.py needs it vocab_processor.save(os.path.join(out_dir, "vocab.pickle")) sess.run(tf.global_variables_initializer()) # Training starts here train_batches = data_helper.batch_iter(list(zip(x_train, y_train)), params['batch_size'], params['num_epochs']) best_accuracy, best_at_step = 0, 0 """Step 6: train the cnn model with x_train and y_train (batch by batch)""" for train_batch in train_batches: x_train_batch, y_train_batch = zip(*train_batch) train_step(x_train_batch, y_train_batch) current_step = tf.train.global_step(sess, global_step) """Step 6.1: evaluate the model with x_dev and y_dev (batch by batch)""" if current_step % params['evaluate_every'] == 0: dev_batches = data_helper.batch_iter( list(zip(x_dev, y_dev)), params['batch_size'], 1) total_dev_correct = 0 for dev_batch in dev_batches: x_dev_batch, y_dev_batch = zip(*dev_batch) num_dev_correct = dev_step(x_dev_batch, y_dev_batch) total_dev_correct += num_dev_correct dev_accuracy = float(total_dev_correct) / len(y_dev) logging.critical( 'Accuracy on dev set: {}'.format(dev_accuracy)) """Step 6.2: save the model if it is the best based on accuracy on dev set""" if dev_accuracy >= best_accuracy: best_accuracy, best_at_step = dev_accuracy, current_step saving_path = "" path = saver.save(sess, checkpoint_prefix, global_step=current_step) logging.critical('Saved model at {} at step {}'.format( path, best_at_step)) logging.critical( 'Best accuracy is {} at step {}'.format( best_accuracy, best_at_step)) checkpointFile = open( './trained_model_' + model_folder_name + '/checkpoints/checkpoint', 'w') checkpointFile.write( "model_checkpoint_path: \"model-" + str(best_at_step) + "\"") checkpointFile.close() """Step 7: predict x_test (batch by batch)""" test_batches = data_helper.batch_iter(list(zip(x_test, y_test)), params['batch_size'], 1) total_test_correct = 0 for test_batch in test_batches: x_test_batch, y_test_batch = zip(*test_batch) num_test_correct = dev_step(x_test_batch, y_test_batch) total_test_correct += num_test_correct test_accuracy = float(total_test_correct) / len(y_test) logging.critical( 'Accuracy on test set is {} based on the best model {}'.format( test_accuracy, path)) logging.critical('The training is complete')
def train_cnn(): """Training CNN model.""" # Load sentences, labels, and training parameters logger.info('✔︎ Loading data...') logger.info('✔︎ Training data processing...') train_data = data_helpers.load_data_and_labels(FLAGS.training_data_file, FLAGS.embedding_dim) logger.info('✔︎ Validation data processing...') validation_data = data_helpers.load_data_and_labels( FLAGS.validation_data_file, FLAGS.embedding_dim) logger.info('Recommand padding Sequence length is: {}'.format( FLAGS.pad_seq_len)) logger.info('✔︎ Training data padding...') x_train_front, x_train_behind, y_train = data_helpers.pad_data( train_data, FLAGS.pad_seq_len) logger.info('✔︎ Validation data padding...') x_validation_front, x_validation_behind, y_validation = data_helpers.pad_data( validation_data, FLAGS.pad_seq_len) # Build vocabulary VOCAB_SIZE = data_helpers.load_vocab_size(FLAGS.embedding_dim) pretrained_word2vec_matrix = data_helpers.load_word2vec_matrix( VOCAB_SIZE, FLAGS.embedding_dim) # Build a graph and cnn object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=FLAGS.pad_seq_len, num_classes=y_train.shape[1], vocab_size=VOCAB_SIZE, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge( [loss_summary, acc_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter( validation_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Initialize all variables sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) def train_step(x_batch_front, x_batch_behind, y_batch): """A single training step""" feed_dict = { cnn.input_x_front: x_batch_front, cnn.input_x_behind: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() logger.info("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def validation_step(x_batch_front, x_batch_behind, y_batch, writer=None): """Evaluates model on a validation set""" feed_dict = { cnn.input_x_front: x_batch_front, cnn.input_x_behind: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, scores, predictions, num_correct, \ loss, accuracy, recall, precision, f1, auc, topKPreds, = sess.run( [global_step, validation_summary_op, cnn.scores, cnn.predictions, cnn.num_correct, cnn.loss, cnn.accuracy, cnn.recall, cnn.precision, cnn.F1, cnn.AUC, cnn.topKPreds], feed_dict) time_str = datetime.datetime.now().isoformat() logger.info( "{}: step {}, loss {:g}, acc {:g}, " "recall {:g}, precision {:g}, f1 {:g}, AUC {}".format( time_str, step, loss, accuracy, recall, precision, f1, auc)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter( list(zip(x_train_front, x_train_behind, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch_front, x_batch_behind, y_batch = zip(*batch) train_step(x_batch_front, x_batch_behind, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") validation_step(x_validation_front, x_validation_behind, y_validation, writer=validation_summary_writer) if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info( "✔︎ Saved model checkpoint to {}\n".format(path)) logger.info("✔︎ Done.")
def train_cnn(): """Step 0: load sentences, labels, and training parameters""" train_file = 'Data/iseardataset.csv' x_raw, y_raw, df, labels, embedding_mat = data_helper.load_data_and_labels( train_file) parameter_file = './parameters.json' params = json.loads(open(parameter_file).read()) """Step 1: pad each sentence to the same length and map each word to an id""" max_document_length = max([len(x.split(' ')) for x in x_raw]) logging.info( 'The maximum length of all sentences: {}'.format(max_document_length)) vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) x = np.array(list(vocab_processor.fit_transform(x_raw))) y = np.array(y_raw) # print x.shape """Step 2: split the original dataset into train and test sets""" x_, x_test, y_, y_test = train_test_split(x, y, test_size=0.2, random_state=42) """Step 3: shuffle the train set and split the train set into train and dev sets""" shuffle_indices = np.random.permutation(np.arange(len(y_))) x_shuffled = x_[shuffle_indices] y_shuffled = y_[shuffle_indices] x_train, x_dev, y_train, y_dev = train_test_split(x_shuffled, y_shuffled, test_size=0.2) """Step 4: save the labels into labels.json since predict.py needs it""" with open('./labels.json', 'w') as outfile: json.dump(labels, outfile, indent=4) logging.info('x_train: {}, x_dev: {}, x_test: {}'.format( len(x_train), len(x_dev), len(x_test))) logging.info('y_train: {}, y_dev: {}, y_test: {}'.format( len(y_train), len(y_dev), len(y_test))) """Step 5: build a graph and cnn object""" graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=9000, embedding_size=params['embedding_dim'], filter_sizes=list( map(int, params['filter_sizes'].split(","))), num_filters=params['num_filters'], embedding_mat=embedding_mat, l2_reg_lambda=params['l2_reg_lambda']) global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "trained_model_" + timestamp)) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.all_variables()) # One training step: train the model with one batch def train_step(x_batch, y_batch): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: params['dropout_keep_prob'] } _, step, loss, acc = sess.run( [train_op, global_step, cnn.loss, cnn.accuracy], feed_dict) # One evaluation step: evaluate the model with one batch def dev_step(x_batch, y_batch): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, loss, acc, num_correct = sess.run( [global_step, cnn.loss, cnn.accuracy, cnn.num_correct], feed_dict) return num_correct # Save the word_to_id map since predict.py needs it vocab_processor.save(os.path.join(out_dir, "vocab.pickle")) sess.run(tf.initialize_all_variables()) print "Loading Embeddings !" initW = data_helper.load_embedding_vectors( vocab_processor.vocabulary_) sess.run(cnn.W.assign(initW)) print "Loaded Embeddings !" # Training starts here train_batches = data_helper.batch_iter(list(zip(x_train, y_train)), params['batch_size'], params['num_epochs']) best_accuracy, best_at_step = 0, 0 """Step 6: train the cnn model with x_train and y_train (batch by batch)""" for train_batch in train_batches: if len(train_batch) == 0: continue x_train_batch, y_train_batch = zip(*train_batch) train_step(x_train_batch, y_train_batch) current_step = tf.train.global_step(sess, global_step) """Step 6.1: evaluate the model with x_dev and y_dev (batch by batch)""" if current_step % params['evaluate_every'] == 0: dev_batches = data_helper.batch_iter( list(zip(x_dev, y_dev)), params['batch_size'], 1) total_dev_correct = 0 for dev_batch in dev_batches: if len(dev_batch) == 0: continue x_dev_batch, y_dev_batch = zip(*dev_batch) num_dev_correct = dev_step(x_dev_batch, y_dev_batch) total_dev_correct += num_dev_correct dev_accuracy = float(total_dev_correct) / len(y_dev) logging.critical( 'Accuracy on dev set: {}'.format(dev_accuracy)) """Step 6.2: save the model if it is the best based on accuracy of the dev set""" if dev_accuracy >= best_accuracy: best_accuracy, best_at_step = dev_accuracy, current_step path = saver.save(sess, checkpoint_prefix, global_step=current_step) logging.critical('Saved model {} at step {}'.format( path, best_at_step)) logging.critical('Best accuracy {} at step {}'.format( best_accuracy, best_at_step)) """Step 7: predict x_test (batch by batch)""" test_batches = data_helper.batch_iter(list(zip(x_test, y_test)), params['batch_size'], 1) total_test_correct = 0 for test_batch in test_batches: if len(test_batch) == 0: continue print "Non Zero Length" x_test_batch, y_test_batch = zip(*test_batch) num_test_correct = dev_step(x_test_batch, y_test_batch) total_test_correct += num_test_correct test_accuracy = float(total_test_correct) / len(y_test) train_batches = data_helper.batch_iter(list(zip(x_train, y_train)), params['batch_size'], 1) total_train_correct = 0 for train_batch in train_batches: if len(train_batch) == 0: continue print "Non Zero Length" x_train_batch, y_train_batch = zip(*train_batch) num_test_correct = dev_step(x_train_batch, y_train_batch) total_train_correct += num_test_correct train_accuracy = float(total_train_correct) / len(y_train) print 'Accuracy on test set is {} based on the best model'.format( test_accuracy) print 'Accuracy on train set is {} based on the best model'.format( train_accuracy) # logging.critical('Accuracy on test set is {} based on the best model {}'.format(test_accuracy, path)) logging.critical('The training is complete')
def cnn_train(training_r, training_w): assert os.path.exists(training_r) and os.path.exists(training_w) VOCAB_DIR = os.path.join(os.path.dirname(training_r), "..") MODEL_DIR = os.path.join(os.path.dirname(training_r), "..", "runs") print "Loading data..." x_train_r, x_train_w, y_train_r, y_train_w = data_helpers.load_data_and_labels( training_r, training_w) # Restore dictionary print "Restoring vocab..." vocab_path = os.path.join(VOCAB_DIR, "vocab") try: assert os.path.exists(vocab_path) vocab_processor = learn.preprocessing.VocabularyProcessor.restore( vocab_path) print "vocab_lenght:", len(vocab_processor.vocabulary_) except Exception as e: print "Failed to restore vocab" traceback.print_exc() print "PreProcessing data..." # Map words to int vector x_train_r = np.array(list(vocab_processor.transform(x_train_r))) y_train_r = np.array(y_train_r) x_train_w = np.array(list(vocab_processor.transform(x_train_w))) y_train_w = np.array(y_train_w) print "Positive/Negative {:d}/{:d}".format(len(y_train_r), len(y_train_w)) # Add priority for right data y_train_r = y_train_r * config_.right_weight # multiply the weight of the right data # Training data x_train = np.array(list(x_train_r) + list(x_train_w)) y_train = np.array(list(y_train_r) + list(y_train_w)) # Training # ================================================== print "Start training..." start_train = time.time() with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=config_.allow_soft_placement, log_device_placement=config_.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train_r.shape[1], num_classes=y_train_r.shape[1], vocab_size=max(config_.max_vocab_keep, len(vocab_processor.vocabulary_)), embedding_size=config_.embedding_dim, filter_sizes=list( map(int, config_.filter_sizes.split(","))), num_filters=config_.num_filters, l2_reg_lambda=config_.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) tf.add_to_collection('train_op', train_op) # Output directory for models and summaries out_dir = os.path.abspath(MODEL_DIR) #if os.path.exists(out_dir): # gfile.DeleteRecursively(out_dir) print "Writing to {}\n".format(out_dir) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=config_.num_checkpoints) # Write vocabulary #vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: config_.dropout_keep_prob, } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.now().isoformat() print "{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy) train_summary_writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), config_.batch_size, config_.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % config_.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print "Saved model checkpoint to {}\n".format(path) end_train = time.time() print 'Train costs', end_train - start_train
def execute(dataset, trainingSection): # # Set up parameters # In[2]: trainDataDir = '/Users/SamZhang/Documents/Capstone/dataset/' + dataset + '/train' # In[3]: # Data loading params tf.flags.DEFINE_float("dev_sample_percentage", .2, "Percentage of the training data to use for validation") tf.flags.DEFINE_string("positive_data_file", trainDataDir + "/spam/" + str(trainingSection) + '/' + dataset + "_train.spam", "Data source for the positive data.") tf.flags.DEFINE_string("negative_data_file", trainDataDir + "/ham/" + str(trainingSection) + '/' + dataset + "_train.ham", "Data source for the negative data.") # Model Hyperparameters tf.flags.DEFINE_integer("embedding_dim", 128, "Dimensionality of character embedding (default: 128)") tf.flags.DEFINE_string("filter_sizes", "3,4,5", "Comma-separated filter sizes (default: '3,4,5')") tf.flags.DEFINE_integer("num_filters", 256, "Number of filters per filter size (default: 128)") tf.flags.DEFINE_float("dropout_keep_prob", 0.5, "Dropout keep probability (default: 0.5)") tf.flags.DEFINE_float("l2_reg_lambda", 0.1, "L2 regularization lambda (default: 0.0)") # Training parameters tf.flags.DEFINE_integer("batch_size", 128, "Batch Size (default: 64)") tf.flags.DEFINE_integer("num_epochs", 30, "Number of training epochs (default: 200)") tf.flags.DEFINE_integer("evaluate_every", 30, "Evaluate model on dev set after this many steps (default: 100)") tf.flags.DEFINE_integer("checkpoint_every", 30, "Save model after this many steps (default: 100)") tf.flags.DEFINE_integer("num_checkpoints", 5, "Number of checkpoints to store (default: 5)") # Misc Parameters tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement") tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") # In[4]: FLAGS = tf.flags.FLAGS FLAGS._parse_flags() print("\nParameters:") for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) print("") # # Process data # In[5]: def processData(): # Data Preparation # ================================================== # Load data print("Loading data...") x_text, y = data_helpers.load_data_and_labels(FLAGS.positive_data_file, FLAGS.negative_data_file) print(x_text[0]) # Build vocabulary max_document_length = max([len(x.split(" ")) for x in x_text]) vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length) x = np.array(list(vocab_processor.fit_transform(x_text))) # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:] del x, y, x_shuffled, y_shuffled print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) return x_train, y_train, x_dev, y_dev, vocab_processor # In[6]: x_train, y_train, x_dev, y_dev, vocab_processor = processData() # # Building CNN # In[7]: with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram("{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", "cnnmodel", dataset, str(trainingSection))) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run( [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter( list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) # # Testing model # In[8]: from sklearn import metrics import csv # In[9]: testDataDir = "/Users/SamZhang/Documents/Capstone/dataset/" + dataset + "/test" # In[10]: tf.flags.DEFINE_string("positive_test_data_file", testDataDir + "/spam/" + str(trainingSection) + '/' + dataset + "_test.spam", "Data source for the positive data.") tf.flags.DEFINE_string("negative_test_data_file", testDataDir + "/ham/" + str(trainingSection) + '/' + dataset + "_test.ham", "Data source for the negative data.") # Eval Parameters tf.flags.DEFINE_string("checkpoint_dir", "/Users/SamZhang/Documents/Capstone/Models/runs/cnnmodel/" + dataset + "/" + str(trainingSection) + "/checkpoints/", "Checkpoint directory from training run") # In[11]: FLAGS = tf.flags.FLAGS FLAGS._parse_flags() print("\nParameters:") for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) print("") # In[12]: x_raw, y_test = data_helpers.load_data_and_labels(FLAGS.positive_test_data_file, FLAGS.negative_test_data_file) y_test = np.argmax(y_test, axis=1) #ham = 0, spam = 1 print(x_raw[0], y_test[0]) # In[13]: vocab_path = os.path.join(FLAGS.checkpoint_dir, "..", "vocab") vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path) x_test = np.array(list(vocab_processor.transform(x_raw))) # In[14]: checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) # In[15]: graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # Get the placeholders from the graph by name input_x = graph.get_operation_by_name("input_x").outputs[0] # input_y = graph.get_operation_by_name("input_y").outputs[0] dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0] # Tensors we want to evaluate predictions = graph.get_operation_by_name("output/predictions").outputs[0] # Generate batches for one epoch batches = data_helpers.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False) # Collect the predictions here all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0}) all_predictions = np.concatenate([all_predictions, batch_predictions]) # Print accuracy if y_test is defined if y_test is not None: correct_predictions = float(sum(all_predictions == y_test)) print("Total number of test examples: {}".format(len(y_test))) print("Accuracy: {:g}".format(correct_predictions/float(len(y_test)))) # In[16]: # Save the evaluation to a csv title = np.column_stack(('text', 'prediction', 'label')) predictions_human_readable = np.column_stack((np.array(x_raw), all_predictions, y_test)) out_path = os.path.join(FLAGS.checkpoint_dir, "..", "prediction.csv") print("Saving evaluation to {0}".format(out_path)) with open(out_path, 'w') as f: csv.writer(f).writerows(title) csv.writer(f).writerows(predictions_human_readable) # # Evaluation and Noise Reduction # In[17]: import ES_interface as esi esi.metric(dataset + '_' + str(trainingSection), out_path)
def main(): tf.flags.DEFINE_float( "dev_sample_percentage", .2, "Percentage of the training data to use for validation") tf.flags.DEFINE_string("positive_data_file", "./data/rt-polaritydata/rt-polarity.pos", "Data source for the positive data.") tf.flags.DEFINE_string("negative_data_file", "./data/rt-polaritydata/rt-polarity.neg", "Data source for the negative data.") # Model Hyperparameters tf.flags.DEFINE_integer( "embedding_dim", 128, "Dimensionality of character embedding (default: 128)") tf.flags.DEFINE_string("filter_sizes", "3,4,5", "Comma-separated filter sizes (default: '3,4,5')") tf.flags.DEFINE_integer( "num_filters", 128, "Number of filters per filter size (default: 128)") tf.flags.DEFINE_float("dropout_keep_prob", 0.5, "Dropout keep probability (default: 0.5)") tf.flags.DEFINE_float("l2_reg_lambda", 0.0, "L2 regularization lambda (default: 0.0)") # Training parameters tf.flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)") tf.flags.DEFINE_integer("num_epochs", 20, "Number of training epochs (default: 200)") tf.flags.DEFINE_integer( "evaluate_every", 100, "Evaluate model on dev set after this many steps (default: 100)") tf.flags.DEFINE_integer("checkpoint_every", 100, "Save model after this many steps (default: 100)") # Misc Parameters tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement") tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") FLAGS = tf.flags.FLAGS FLAGS._parse_flags() user_reviews = [] user_ratings = [] with open('ratings.txt', 'rb') as fp: user_ratings = np.array(pickle.load(fp)) with open('reviews.txt', 'rb') as fp: user_reviews = np.array(pickle.load(fp)) user_ratings = format_rating(user_ratings) x_text = user_reviews y = user_ratings # Build vocabulary max_document_length = max([len(x.split(" ")) for x in x_text]) vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) x = np.array(list(vocab_processor.fit_transform(x_text))) # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation #dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) val_sample_index = int(FLAGS.dev_sample_percentage * float(len(x_shuffled))) accuracy_list = [] for i in range(5): #x_train, x_val = x_shuffled[:i*val_sample_index]+x_shuffled[i*val_sample_index+val_sample_index:], x_shuffled[dev_sample_index:] #y_train, y_val = y_shuffled[i*val_sample_index:i*val_sample_index+val_sample_index], y_shuffled[i*val_sample_index:i*val_sample_index+val_sample_index] y_val = y_shuffled[i * val_sample_index:i * val_sample_index + val_sample_index] y_train = np.array( list(y_shuffled[:i * val_sample_index]) + list(y_shuffled[i * val_sample_index + val_sample_index:])) x_val = x_shuffled[i * val_sample_index:i * val_sample_index + val_sample_index] x_train = np.array( list(x_shuffled[:i * val_sample_index]) + list(x_shuffled[i * val_sample_index + val_sample_index:])) print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_val))) # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.histogram_summary( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.scalar_summary( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.merge_summary(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.scalar_summary("loss", cnn.loss) acc_summary = tf.scalar_summary("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.merge_summary( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.train.SummaryWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.merge_summary([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.train.SummaryWriter( dev_summary_dir, sess.graph) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables sess.run(tf.initialize_all_variables()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print((i + 1), "{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print((i + 1), "{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) return accuracy # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_val, y_val, writer=dev_summary_writer) print("") # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it #checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_dir = "checkpoints_" + str(i) + "/" if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver() checkpoint_prefix = os.path.join(checkpoint_dir, "model") saver.save(sess, checkpoint_prefix, global_step=current_step) accuracy_list.append( dev_step(x_val, y_val, writer=dev_summary_writer)) print("After cross validation ") print("Accuracy=", np.mean(accuracy_list))
def train(): with tf.device('/cpu:0'): x_text, pos1, pos2, y = data_helpers.load_hw_data_and_labels( FLAGS.train_dir) # Build vocabulary # Example: x_text[3] = "A misty <e1>ridge</e1> uprises from the <e2>surge</e2>." # ['a misty ridge uprises from the surge <UNK> <UNK> ... <UNK>'] # => # [27 39 40 41 42 1 43 0 0 ... 0] # dimension = FLAGS.max_sentence_length text_vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor( FLAGS.max_sentence_length) text_vec = np.array(list(text_vocab_processor.fit_transform(x_text))) print("Text Vocabulary Size: {:d}".format( len(text_vocab_processor.vocabulary_))) # Example: pos1[3] = [-2 -1 0 1 2 3 4 999 999 999 ... 999] # [95 96 97 98 99 100 101 999 999 999 ... 999] # => # [11 12 13 14 15 16 21 17 17 17 ... 17] # dimension = MAX_SENTENCE_LENGTH pos_vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor( FLAGS.max_sentence_length) pos_vocab_processor.fit(pos1 + pos2) pos1_vec = np.array(list(pos_vocab_processor.transform(pos1))) pos2_vec = np.array(list(pos_vocab_processor.transform(pos2))) print("Position Vocabulary Size: {:d}".format( len(pos_vocab_processor.vocabulary_))) x = np.array([list(i) for i in zip(text_vec, pos1_vec, pos2_vec)]) print("x = {0}".format(x.shape)) print("y = {0}".format(y.shape)) print("") # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[ dev_sample_index:] x_dev = np.array(x_dev).transpose((1, 0, 2)) y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[ dev_sample_index:] print("Train/Dev split: {:d}/{:d}\n".format(len(y_train), len(y_dev))) with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=x_train.shape[2], num_classes=y_train.shape[1], text_vocab_size=len(text_vocab_processor.vocabulary_), text_embedding_size=FLAGS.text_embedding_dim, pos_vocab_size=len(pos_vocab_processor.vocabulary_), pos_embedding_size=FLAGS.position_embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize( cnn.loss, global_step=global_step) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary text_vocab_processor.save(os.path.join(out_dir, "text_vocab")) pos_vocab_processor.save(os.path.join(out_dir, "position_vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) # Pre-trained word2vec if FLAGS.word2vec: # initial matrix with random uniform initW = np.random.uniform( -0.25, 0.25, (len(text_vocab_processor.vocabulary_), FLAGS.text_embedding_dim)) # load any vectors from the word2vec print("Load word2vec file {0}".format(FLAGS.word2vec)) with open(FLAGS.word2vec, "rb") as f: header = f.readline() print('header :', header) vocab_size, layer1_size = map(int, header.split()) binary_len = np.dtype('float32').itemsize * layer1_size for line in range(vocab_size): word = [] while True: ch = f.read(1).decode('latin-1') print('ch :', ch) if ch == ' ': word = ''.join(word) break if ch != '\n': word.append(ch) idx = text_vocab_processor.vocabulary_.get(word) if idx != 0: initW[idx] = np.fromstring(f.read(binary_len), dtype='float32') else: f.read(binary_len) print('initW[0] :', initW[0], initW[1]) sess.run(cnn.W_text.assign(initW)) print("Success to load pre-trained word2vec model!\n") # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) x_batch = np.array(x_batch).transpose((1, 0, 2)) # Train feed_dict = { cnn.input_text: x_batch[0], cnn.input_pos1: x_batch[1], cnn.input_pos2: x_batch[2], cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) train_summary_writer.add_summary(summaries, step) # Training log display if step % FLAGS.display_every == 0: time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) # Evaluation if step % FLAGS.evaluate_every == 0: print("\nEvaluation:") feed_dict = { cnn.input_text: x_dev[0], cnn.input_pos1: x_dev[1], cnn.input_pos2: x_dev[2], cnn.input_y: y_dev, cnn.dropout_keep_prob: 1.0 } summaries, loss, accuracy, predictions = sess.run([ dev_summary_op, cnn.loss, cnn.accuracy, cnn.predictions ], feed_dict) dev_summary_writer.add_summary(summaries, step) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) print( "(2*9+1)-Way Macro-Average F1 Score (excluding Other): {:g}\n" .format( f1_score(np.argmax(y_dev, axis=1), predictions, labels=np.array(range(1, 19)), average="macro"))) # Model checkpoint if step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=step) print("Saved model checkpoint to {}\n".format(path))
def main(_): # Load data print("Loading data...") x_, y = data_helpers.load_data_and_labels(FLAGS.train_file) train_int_to_vab, train_to_int = data_helpers.cret_dict(x_) #保存对应的词和词索引 print(train_int_to_vab) #存储所有字的文件,以便测试加载 pickle.dump(train_int_to_vab, open('./vocab_index.pkl', 'wb')) print(train_int_to_vab) train_ids = [[ train_to_int.get(term, train_to_int['<UNK>']) for term in line ] for line in x_] x_ = data_helpers.pad_sentences(train_ids, 20) # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x_[shuffle_indices] y = np.array(y) y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[ dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[ dev_sample_index:] print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) y_train = np_utils.to_categorical(y_train) y_dev = np_utils.to_categorical(y_dev) # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(train_int_to_vab), embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
def training(): """Train the model. """ x_train, y_train, x_test, y_test, vocab_size = data_preparation() print('---------------------------------------------') print(x_train.shape) print(y_train.shape) print(x_test.shape) print(y_test.shape) print('---------------------------------------------') with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=vocab_size, embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(','))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name='global_step', trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( '{}/grad/hist'.format(v.name), g) sparsity_summary = tf.summary.scalar( '{}/grad/sparsity'.format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries out_dir = os.path.abspath( os.path.join(os.path.curdir, FLAGS.train, TIMESTAMP)) print('Writing to {}\n'.format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar('loss', cnn.loss) acc_summary = tf.summary.scalar('accuracy', cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, 'summaries', 'train') train_summary_writer = tf.train.SummaryWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, 'summaries', 'dev') dev_summary_writer = tf.train.SummaryWriter( dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists # so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, 'checkpoints')) checkpoint_prefix = os.path.join(checkpoint_dir, 'model') if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables()) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): ''' A single training step ''' feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print('%s: step %8d, loss %2.6f, acc %2.6f\r' % (time_str, step, loss, accuracy), end='') def validate_step(x, y, writer=None): ''' Evaluates model on a dev set ''' size = len(y) if size < FLAGS.batch_size: raise ValueError( "batch size for evals larger than dataset: %d" % size) accuracies = [] for begin in xrange(0, size, FLAGS.batch_size): end = begin + FLAGS.batch_size if end < size: feed_dict = { cnn.input_x: x[begin:end, :], cnn.input_y: y[begin:end, :], cnn.dropout_keep_prob: 1.0 } else: feed_dict = { cnn.input_x: x[-FLAGS.batch_size:, :], cnn.input_y: y[-FLAGS.batch_size:, :], cnn.dropout_keep_prob: 1.0 } summaries, step, accuracy = sess.run( [dev_summary_op, global_step, cnn.accuracy], feed_dict) accuracies.append(accuracy) if writer: writer.add_summary(summaries, step) accuracy = sum(accuracies) / len(accuracies) time_str = datetime.datetime.now().isoformat() print('%s: step %8d, acc %2.6f' % (time_str, step, accuracy)) def class_accuracy_stat(predictions, labels, all_data=False): predictions = predictions.tolist() labels = labels.tolist() with open( os.path.join(FLAGS.train, TIMESTAMP, 'label_to_int.json'), 'r') as f: label_to_int = json.load(f) int_to_label = { label_to_int[key]: key for key in label_to_int.keys() } labels_stat = {} for i in xrange(len(labels)): try: labels_stat[int_to_label[ labels[i]]]['num'] = labels_stat[int_to_label[ labels[i]]]['num'] + 1 if labels_stat[ int_to_label[labels[i]]]['num'] else 1 except: labels_stat[int_to_label[labels[i]]] = {} labels_stat[int_to_label[ labels[i]]]['num'] = labels_stat[int_to_label[ labels[i]]]['num'] + 1 if labels_stat[ int_to_label[labels[i]]].has_key( 'num') else 1 if labels[i] == predictions[i]: labels_stat[int_to_label[labels[i]]][ 'predictions'] = labels_stat[int_to_label[ labels[i]]]['predictions'] + 1 if labels_stat[ int_to_label[labels[i]]].has_key( 'predictions') else 1 for label in labels_stat: if not labels_stat[label].has_key('predictions'): labels_stat[label]['predictions'] = 0 for label in labels_stat: labels_stat[label]['accuracy'] = float( labels_stat[label] ['predictions']) / labels_stat[label]['num'] if all_data: with open( os.path.join(FLAGS.train, TIMESTAMP, 'all_labels_stat.json'), 'w+') as f: json.dump(labels_stat, f) else: with open( os.path.join(FLAGS.train, TIMESTAMP, 'test_labels_stat.json'), 'w+') as f: json.dump(labels_stat, f) def test_step(x, y, writer=None, all_data=False): ''' Evaluates model on a dev set ''' size = len(y) accuracies = [] all_predictions = [] for begin in xrange(0, size, FLAGS.batch_size): end = begin + FLAGS.batch_size if end < size: feed_dict = { cnn.input_x: x[begin:end, :], cnn.input_y: y[begin:end, :], cnn.dropout_keep_prob: 1.0 } else: feed_dict = { cnn.input_x: x[-FLAGS.batch_size:, :], cnn.input_y: y[-FLAGS.batch_size:, :], cnn.dropout_keep_prob: 1.0 } summaries, step, accuracy, batch_predictions = sess.run([ dev_summary_op, global_step, cnn.accuracy, cnn.predictions ], feed_dict) all_predictions = np.concatenate( [all_predictions, batch_predictions]) accuracies.append(accuracy) if writer: writer.add_summary(summaries, step) class_accuracy_stat(all_predictions, np.argmax(y, axis=1), all_data) accuracy = sum(accuracies) / len(accuracies) time_str = datetime.datetime.now().isoformat() print('%s: step %8d, acc %2.6f' % (time_str, step, accuracy)) # Generate batches batches = _batch_iter(x_train, y_train, FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = batch train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print('\nValidation:') validate_step(x_test, y_test, writer=dev_summary_writer) print('') if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print('Saved model checkpoint to {}\n'.format(path)) print('\nTest:') test_step(x_test, y_test, writer=dev_summary_writer) test_step(np.concatenate((x_train, x_test), axis=0), np.concatenate((y_train, y_test), axis=0), writer=dev_summary_writer, all_data=True) print('')
def train_cnn(): """Training CNN model.""" # Load sentences, labels, and training parameters logger.info("✔︎ Loading data...") logger.info("✔︎ Training data processing...") train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.num_classes, FLAGS.embedding_dim, data_aug_flag=False) logger.info("✔︎ Validation data processing...") val_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes, FLAGS.embedding_dim, data_aug_flag=False) logger.info("Recommended padding Sequence length is: {0}".format( FLAGS.pad_seq_len)) logger.info("✔︎ Training data padding...") x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len) logger.info("✔︎ Validation data padding...") x_val, y_val = dh.pad_data(val_data, FLAGS.pad_seq_len) # Build vocabulary VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim) pretrained_word2vec_matrix = dh.load_word2vec_matrix( VOCAB_SIZE, FLAGS.embedding_dim) # Build a graph and cnn object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=FLAGS.pad_seq_len, num_classes=FLAGS.num_classes, vocab_size=VOCAB_SIZE, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, filter_sizes=list( map(int, FLAGS.filter_sizes.split(','))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure with tf.control_dependencies( tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay( learning_rate=FLAGS.learning_rate, global_step=cnn.global_step, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(cnn.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio) train_op = optimizer.apply_gradients( zip(grads, vars), global_step=cnn.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram( "{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries if FLAGS.train_or_restore == 'R': MODEL = input( "☛ Please input the checkpoints model you want to restore, " "it should be like(1490175368): " ) # The model you want to restore while not (MODEL.isdigit() and len(MODEL) == 10): MODEL = input( "✘ The format of your input is illegal, please re-input: " ) logger.info( "✔︎ The format of your input is legal, now loading to next step..." ) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", MODEL)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) else: timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) best_checkpoint_dir = os.path.abspath( os.path.join(out_dir, "bestcheckpoints")) # Summaries for loss loss_summary = tf.summary.scalar("loss", cnn.loss) # Train summaries train_summary_op = tf.summary.merge( [loss_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter( validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True) if FLAGS.train_or_restore == 'R': # Load cnn model logger.info("✔︎ Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) else: if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = "embedding" embedding_conf.metadata_path = FLAGS.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save( sess, os.path.join(out_dir, "embedding", "embedding.ckpt")) current_step = sess.run(cnn.global_step) def train_step(x_batch, y_batch): """A single training step""" feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: True } _, step, summaries, loss = sess.run( [train_op, cnn.global_step, train_summary_op, cnn.loss], feed_dict) logger.info("step {0}: loss {1:g}".format(step, loss)) train_summary_writer.add_summary(summaries, step) def validation_step(x_val, y_val, writer=None): """Evaluates model on a validation set""" batches_validation = dh.batch_iter(list(zip(x_val, y_val)), FLAGS.batch_size, 1) # Predict classes by threshold or topk ('ts': threshold; 'tk': topk) eval_counter, eval_loss = 0, 0.0 eval_pre_tk = [0.0] * FLAGS.top_num eval_rec_tk = [0.0] * FLAGS.top_num eval_F_tk = [0.0] * FLAGS.top_num true_onehot_labels = [] predicted_onehot_scores = [] predicted_onehot_labels_ts = [] predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)] for batch_validation in batches_validation: x_batch_val, y_batch_val = zip(*batch_validation) feed_dict = { cnn.input_x: x_batch_val, cnn.input_y: y_batch_val, cnn.dropout_keep_prob: 1.0, cnn.is_training: False } step, summaries, scores, cur_loss = sess.run([ cnn.global_step, validation_summary_op, cnn.scores, cnn.loss ], feed_dict) # Prepare for calculating metrics for i in y_batch_val: true_onehot_labels.append(i) for j in scores: predicted_onehot_scores.append(j) # Predict by threshold batch_predicted_onehot_labels_ts = \ dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold) for k in batch_predicted_onehot_labels_ts: predicted_onehot_labels_ts.append(k) # Predict by topK for top_num in range(FLAGS.top_num): batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk( scores=scores, top_num=top_num + 1) for i in batch_predicted_onehot_labels_tk: predicted_onehot_labels_tk[top_num].append(i) eval_loss = eval_loss + cur_loss eval_counter = eval_counter + 1 if writer: writer.add_summary(summaries, step) eval_loss = float(eval_loss / eval_counter) # Calculate Precision & Recall & F1 (threshold & topK) eval_pre_ts = precision_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_rec_ts = recall_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_F_ts = f1_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') for top_num in range(FLAGS.top_num): eval_pre_tk[top_num] = precision_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_rec_tk[top_num] = recall_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_F_tk[top_num] = f1_score( y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') # Calculate the average AUC eval_auc = roc_auc_score( y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') # Calculate the average PR eval_prc = average_precision_score( y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \ eval_rec_tk, eval_pre_tk, eval_F_tk # Generate batches batches_train = dh.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) num_batches_per_epoch = int( (len(x_train) - 1) / FLAGS.batch_size) + 1 # Training loop. For each batch... for batch_train in batches_train: x_batch_train, y_batch_train = zip(*batch_train) train_step(x_batch_train, y_batch_train) current_step = tf.train.global_step(sess, cnn.global_step) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") eval_loss, eval_auc, eval_prc, \ eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk = \ validation_step(x_val, y_val, writer=validation_summary_writer) logger.info( "All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}" .format(eval_loss, eval_auc, eval_prc)) # Predict by threshold logger.info( "☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F {2:g}" .format(eval_pre_ts, eval_rec_ts, eval_F_ts)) # Predict by topK logger.info("☛ Predict by topK:") for top_num in range(FLAGS.top_num): logger.info( "Top{0}: Precision {1:g}, Recall {2:g}, F {3:g}". format(top_num + 1, eval_pre_tk[top_num], eval_rec_tk[top_num], eval_F_tk[top_num])) best_saver.handle(eval_prc, sess, current_step) if current_step % FLAGS.checkpoint_every == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info( "✔︎ Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info( "✔︎ Epoch {0} has finished!".format(current_epoch)) logger.info("✔︎ Done.")
def train_with_dev(x_train, x_dev, y_train, y_dev, vocab_processor): with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): print('modeling...') cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) print("cnn model finished!") # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) + "summaries" out_dir = os.path.abspath( os.path.join(os.path.curdir, "model", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # create saver for model saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # initialize model ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir) if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: print("Initialize model with fresh parameters.") sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() if step % 100 == 0: print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... debug_step = -1 # 100 for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") if current_step % FLAGS.checkpoint_every == 0: checkpoint_path = os.path.join(FLAGS.model_dir, "textcnn.ckpt") path = saver.save(sess, checkpoint_path, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) if current_step == debug_step: print('training is over caused by debug_step') break
def train_cnn(training_config_file=None): params = json.loads(open(training_config_file).read()) # Data Preparation # ================================================== # Load data print("Loading data...") # x_text, y = data_helpers.load_data_and_labels(params["positive_data_file"], params["negative_data_file"]) x_text, y = data_helpers.load_text_and_path_label(params["file_level_train_file"], params["file_level_label"]) # Build vocabulary max_document_length = max([len(x.split(" ")) for x in x_text]) vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length) x = np.array(list(vocab_processor.fit_transform(x_text))) # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation dev_sample_index = -1 * int(params["dev_sample_percentage"] * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:] del x, y, x_shuffled, y_shuffled print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=params["allow_soft_placement"], log_device_placement=params["log_device_placement"]) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=params["embedding_dim"], filter_sizes=list(map(int, params["filter_sizes"].split(","))), num_filters=params["num_filters"], l2_reg_lambda=params["l2_reg_lambda"]) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram("{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=params["num_checkpoints"]) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: params["dropout_keep_prob"] } _, step, summaries, loss, accuracy = sess.run( [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter( list(zip(x_train, y_train)), params["batch_size"], params["num_epochs"]) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % params["evaluate_every"] == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") if current_step % params["checkpoint_every"] == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
#print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_test))) #Training #=============================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement = allow_soft_placement, log_device_placement = log_device_placement ) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_size = x_train.shape[1], num_classes = y_train.shape[1], vocab_size = len(vocab.vocabulary_), embedding_size = embedding_dim, filter_sizes = filter_sizes, num_filters = num_filters, l2_reg_lambda = l2_reg_lambda ) #Define training procedure global_step = tf.Variable(0, name="global_step", trainable=False) #print(global_step) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) #记录梯度变化和稀疏度(可选) grad_summaries = [] for g, v in grads_and_vars: if g is not None:
def train(x_train, y_train, vocab_processor, x_dev, y_dev): # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) #max_to_keep:lưu num_checkpoints checkpoints gần nhất để lưu saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab.txt")) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, loss, accuracy = sess.run( [train_op, global_step, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) # train_summary_writer.add_summary(summaries, step) #if step == 801: # return 0 #else: # return 1 def dev_step(x_batch, y_batch): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, loss, accuracy = sess.run( [global_step, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs, shuffle=True) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) if (train_step(x_batch, y_batch) == 0): break current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") dev_step(x_dev, y_dev) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=embedding_dimension, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) # optimizer = tf.train.AdamOptimizer(1e-3) # default # optimizer = tf.train.RMSPropOptimizer(0.001) optimizer = tf.train.AdadeltaOptimizer(1) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
def main(unused_argv): if FLAGS.job_name is None or FLAGS.job_name == '': raise ValueError('Must specify an explicit job_name !') else: print ('job_name : ' + FLAGS.job_name) if FLAGS.task_index is None or FLAGS.task_index == '': raise ValueError('Must specify an explicit task_index!') else: print ('task_index : ' + str(FLAGS.task_index)) ps_spec = FLAGS.ps_hosts.split(',') worker_spec = FLAGS.worker_hosts.split(',') num_worker = len(worker_spec) print ("Number of worker = " + str(num_worker)) print ("ps_spec = ") print(*ps_spec) print ("worker_spec = ") print(*worker_spec) cluster = tf.train.ClusterSpec({'ps': ps_spec, 'worker': worker_spec}) print ("After defining Cluster") print ("Job name = " + FLAGS.job_name) print ("task index = " + str(FLAGS.task_index)) # try: server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) print ("After defining server") if FLAGS.job_name == 'ps': print("Parameter Server is executed") server.join() elif FLAGS.job_name == "worker": print("Parameter Server is executed") with tf.device(tf.train.replica_device_setter( worker_device="/job:worker/task:%d"% FLAGS.task_index, cluster=cluster)): is_chief = (FLAGS.task_index == 0) # Data Preparation # ================================================== # Load data print("Loading data...") x_text, y_label = data_helpers.load_data_and_labels(FLAGS.data_file) # Build vocabulary max_document_length = max([len(x.split(" ")) for x in x_text]) vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length) x = np.array(list(vocab_processor.fit_transform(x_text))) y = np.array(y_label) # Randomly shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) print(type(x),type(y)) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:] print (y_train.shape) print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training # ================================================== tf.MaxAcc = 0.1 def copymax(path): shutil.copy(path, "{}.backup".format(path)) cnn = TextCNN( sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.train.get_or_create_global_step() optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = FLAGS.out_dir print("Writing to {}\n".format(out_dir)) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") MaxAcc_prefi=os.path.join(checkpoint_dir, "MAXACCmodel") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) # Initialize all variables session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) init_op=tf.global_variables_initializer() sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0), logdir=out_dir, init_op=init_op, saver=saver, global_step=global_step ) sess = sv.prepare_or_wait_for_session(server.target, config=session_conf) # Generate batches batches = data_helpers.batch_iter( list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) _, current_step, loss, accuracy = sess.run( [train_op, global_step, cnn.loss, cnn.accuracy], feed_dict={cnn.input_x: x_batch,cnn.input_y: y_batch,cnn.dropout_keep_prob: FLAGS.dropout_keep_prob}) time_str = datetime.datetime.now().isoformat() if current_step % 10 ==0: print("{}: step {}, loss {:g}, acc {:g}".format(time_str, current_step, loss, accuracy)) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") loss, accuracy = sess.run( [cnn.loss, cnn.accuracy], feed_dict={cnn.input_x: x_batch,cnn.input_y: y_batch,cnn.dropout_keep_prob: 1.0}) time_str = datetime.datetime.now().isoformat() result = "{}: step {}, loss {:g}, acc {:g}".format(time_str, current_step, loss, accuracy) print(result) with open(os.path.join(out_dir, "result"), 'a+') as f: f.write("{}\n".format(result)) if tf.MaxAcc<accuracy: tf.MaxAcc=accuracy ifsave=True else: ifsave=False print("Max acc:{}".format(tf.MaxAcc)) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) if ifsave: path = saver.save(sess, MaxAcc_prefi, None) copymax("{}.data-00000-of-00001".format(path)) copymax("{}.index".format(path)) copymax("{}.meta".format(path))
def make_graph(x_train, y_train, vocab_processor, x_dev, y_dev): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=68, filter_sizes=[3, 4, 5], num_filters=63, l2_reg_lambda=0.0) with tf.Session() as sess: graph = tf.get_default_graph() print(tf.trainable_variables()) #sess.run(tf.global_variables_initializer()) #print(sess.run(tf.get_default_graph().get_tensor_by_name('output/W:0')).shape) restore_variables = { v.name.split(':')[0]: v for v in tf.trainable_variables() if (v.name.find('conv-maxpool') != -1) or ( v.name.find('embedding') != -1) } #restore_variables = {v.name.split(':')[0]:v for v in tf.trainable_variables() if (v.name.find('conv-maxpool')!=-1) or (v.name.find('embedding')!=-1)} print(restore_variables) saver = tf.train.Saver(restore_variables, max_to_keep=5) saver.restore( sess, tf.train.latest_checkpoint( './pruned_data_by_variance_filters_corporate_round_2/checkpoints/' )) # print([n.name for n in tf.get_default_graph().as_graph_def().node]) global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) tf.add_to_collection("optimizer", train_op) sess.run(tf.variables_initializer(optimizer.variables())) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "pruned_by_variance", timestamp)) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(restore_variables, max_to_keep=5) #sess.run(tf.variables_initializer(optimizer.variables())) sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 0.25 } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g},".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_helpers.batch_iter(list(zip(x_train, y_train)), 64, 10) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % 100 == 0: print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) print("") path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=options.allow_soft_placement, log_device_placement=options.log_device_placement) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), embedding_size=EMBEDDING_DIM, filter_sizes=list(map(int, FILTER_SIZE.split(","))), num_filters=NUM_FILTERS, l2_reg_lambda=L2_REG) global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(LEARNING_RATE) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) timestamp = str(int(time.time())) out_dir = options.checkpoint_dir if not os.path.exists(out_dir): os.makedirs(out_dir) print("Writing to {}\n".format(out_dir))
def train_cnn(): # Data Preparation # ================================================== if FLAGS.init_embedding_path is not None: embedding = np.load(FLAGS.init_embedding_path) print("Using pre-trained word embedding which shape is {}\n".format( embedding.shape)) FLAGS.vocab_size = embedding.shape[0] FLAGS.embedding_size = embedding.shape[1] if FLAGS.init_model_path is not None: assert os.path.isdir( FLAGS.init_model_path), "init_model_path must be a directory\n" ckpt = tf.train.get_checkpoint_state(FLAGS.init_model_path) assert ckpt, "No checkpoint found in {}\n".format( FLAGS.init_model_path) assert ckpt.model_checkpoint_path, "No model_checkpoint_path found in checkpoint\n" # Create root directory timestamp = str(int(time.time())) root_dir = os.path.join(os.path.curdir, 'runs', 'textcnn', 'trained_result_' + timestamp) os.makedirs(root_dir) # Load data print("Loading data...\n") x_data = np.loadtxt(FLAGS.x_data_file) print(x_data.shape) x_data = x_data.reshape(20480, 30, 20) x_data = x_data.reshape(20480, 600) y_data = np.loadtxt(FLAGS.y_data_file) print(y_data.shape) print("data load finished") # Split dataset x_train, x_test, y_train, y_test = train_test_split( x_data, y_data, test_size=FLAGS.test_size, stratify=y_data, random_state=0) x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=0) # Training # ================================================== with tf.Graph().as_default(): tf_config = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth with tf.Session(config=tf_config).as_default() as sess: cnn = TextCNN(vocab_size=FLAGS.vocab_size, embedding_size=FLAGS.embedding_size, sequence_length=FLAGS.sequence_length, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, num_classes=FLAGS.num_classes, learning_rate=FLAGS.learning_rate, grad_clip=FLAGS.grad_clip, l2_reg_lambda=FLAGS.l2_reg_lambda) # Output directory for models and summaries out_dir = os.path.abspath(root_dir) print("Writing to {}...\n".format(out_dir)) # Summaries for loss and accuracy tf.summary.scalar("loss", cnn.loss) tf.summary.scalar("accuracy", cnn.accuracy) merged_summary = tf.summary.merge_all() # Summaries dictionary train_summary_dir = os.path.join(out_dir, 'summaries', 'train') val_summary_dir = os.path.join(out_dir, 'summaries', 'val') train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) val_summary_writer = tf.summary.FileWriter(val_summary_dir, sess.graph) # Checkpoint directory, will not create itself checkpoint_dir = os.path.abspath( os.path.join(out_dir, 'checkpoints')) checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt') if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) # Initialize all variables sess.run(tf.global_variables_initializer()) # Using pre-trained word embedding # if FLAGS.init_embedding_path is not None: # sess.run(cnn.embedding.assign(embedding)) # del embedding # Continue training from saved model if FLAGS.init_model_path is not None: saver.restore(sess, ckpt.model_checkpoint_path) # Training start print("Start training...\n") best_at_step = 0 best_val_accuracy = 0 #**************************************** cnn_feature = [] for epoch in range(FLAGS.num_epochs): # Generate train batches train_batches = data_utils.batch_iter( list(zip(x_train, y_train)), FLAGS.batch_size) start = time.time() for batch in train_batches: # Training model on x_batch and y_batch x_batch, y_batch = zip(*batch) feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: True } _, global_step, train_summaries, train_loss, train_accuracy = sess.run( [ cnn.train_op, cnn.global_step, merged_summary, cnn.loss, cnn.accuracy ], feed_dict=feed_dict) # Evaluates model on val set if global_step % FLAGS.evaluate_every == 0: end = time.time() train_summary_writer.add_summary( train_summaries, global_step) feed_dict = { cnn.input_x: x_val, cnn.input_y: y_val, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: False } val_summaries, val_loss, val_accuracy = sess.run( [merged_summary, cnn.loss, cnn.accuracy], feed_dict=feed_dict) val_summary_writer.add_summary(val_summaries, global_step) print( "Epoch: {}, global step: {}, training speed: {:.3f}sec/batch" .format(epoch, global_step, (end - start) / FLAGS.evaluate_every)) print( "train loss: {:.3f}, train accuracy: {:.3f}, val loss: {:.3f}, val accuracy: {:.3f}\n" .format(train_loss, train_accuracy, val_loss, val_accuracy)) # If improved, save the model if val_accuracy > best_val_accuracy: print( "Get a best val accuracy at step {}, model saving...\n" .format(global_step)) saver.save(sess, checkpoint_prefix, global_step=global_step) best_val_accuracy = val_accuracy best_at_step = global_step start = time.time() # Rename the checkpoint best_model_prefix = checkpoint_prefix + '-' + str(best_at_step) os.rename(best_model_prefix + '.index', os.path.join(checkpoint_dir, 'best_model.index')) os.rename(best_model_prefix + '.meta', os.path.join(checkpoint_dir, 'best_model.meta')) os.rename( best_model_prefix + '.data-00000-of-00001', os.path.join(checkpoint_dir, 'best_model.data-00000-of-00001')) # Testing on test set print( "\nTraining complete, testing the best model on test set...\n") saver.restore(sess, os.path.join(checkpoint_dir, 'best_model')) feed_dict = { cnn.input_x: x_test, cnn.input_y: y_test, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: False } # y_logits, test_accuracy = sess.run([cnn.logits, cnn.accuracy], feed_dict=feed_dict) y_pred, test_accuracy = sess.run([cnn.predictions, cnn.accuracy], feed_dict=feed_dict) print("Testing Accuracy: {:.3f}\n".format(test_accuracy)) y_test_original = np.argmax(y_test, 1) print(y_test_original.shape) # label_transformer = joblib.load(os.path.join(out_dir, 'label_transformer.pkl')) # y_test_original [ '体育','娱乐','家居','彩票','房产','教育','时尚','时政','星座','游戏','社会']= label_transformer.inverse_transform(y_test) # y_logits_original = label_transformer.inverse_transform(y_logits) # print("Precision, Recall and F1-Score:\n\n", classification_report(y_test_original, y_logits_original)) print( "Precision, Recall and F1-Score:\n\n", classification_report(y_test_original, y_pred, target_names=[ '体育', '娱乐', '家居', '彩票', '房产', '教育', '时尚', '时政', '星座', '游戏', '社会' ]))
def main(is_test=False): Config._use_lemma = FLAGS.use_lemma tf.reset_default_graph() tf.set_random_seed(12) with tf.Graph().as_default(): data = load_data() weight = data.weights vocab_size = data.get_vocab_size() train_iterator_initializer = data._train_iterator_initializer validation_iterator_initializer = data._validation_iterator_initializer test_iterator_initializer = data._test_iterator_initializer handle = data.handle next_element = data.next_element session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate, embedding_size=FLAGS.embedding_size, vocab_size=vocab_size, sequence_length=FLAGS.max_length, num_filters=FLAGS.num_filters, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), weight=weight, labes_num=FLAGS.labels_num, output_dimension=FLAGS.output_dimension, next_element=next_element) writer = tf.summary.FileWriter('graphs/text_cnn/learning_rate' + str(cnn._learning_rate)) if not os.path.exists(FLAGS.checkpoint_path): os.makedirs(FLAGS.checkpoint_path) checkpoints_prefix = os.path.join(FLAGS.checkpoint_path, "text_cnn") saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) train_handle = sess.run(data._train_iterator.string_handle()) validation_handle = sess.run( data._validation_iterator.string_handle()) test_handle = sess.run(data._test_iterator.string_handle()) # 是否需要恢复模型 ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path) if ckpt and ckpt.model_checkpoint_path: print("正在从{}加载模型".format(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) def test_step(test_handle): feed_dict = {cnn._keep_prob: 1.0, handle: test_handle} res = sess.run(cnn.predict - 2, feed_dict=feed_dict) data.feed_output(res) def test(): global_step = sess.run(cnn.global_step) sess.run(test_iterator_initializer) while True: try: test_step(test_handle) except tf.errors.OutOfRangeError: data.persist( filename="result_{}.csv".format(global_step)) print("测试结果已经保存, result_{}.csv".format(global_step)) break if is_test: print("正在进行测试") test() print("测试结束") return def train_step(): feed_dict = {cnn._keep_prob: 1.0, handle: train_handle} _, loss, global_step, summary_op, actual_batch_size = sess.run( [ cnn._train_total, cnn._total_loss, cnn.global_step, cnn._summary_op, cnn._actual_batch_size ], feed_dict=feed_dict) writer.add_summary(summary_op, global_step=global_step) return loss, global_step, actual_batch_size def validation_step(): feed_dict = {cnn._keep_prob: 1.0, handle: validation_handle} loss, actual_batch_size, lab, res = sess.run( [ cnn._total_loss, cnn._actual_batch_size, tf.argmax(cnn._label, axis=2) - 2, cnn.predict - 2 ], feed_dict=feed_dict) return loss, actual_batch_size, lab, res def train(total_loss, batches): while True: try: t1 = time.time() loss, step, actual_batch_size = train_step() total_loss += loss batches += 1 delta_t = time.time() - t1 print( "training: step is {}, loss is {}, average_loss is {}, cost {} 秒" .format(step, loss, total_loss / batches, delta_t)) if step > FLAGS.step_bypass_validation and step % FLAGS.step_validation == 0: saver.save(sess, save_path=checkpoints_prefix, global_step=step) average_f1 = validation() if average_f1 > 0.68: test() if step % FLAGS.step_validation == 0: saver.save(sess, save_path=checkpoints_prefix, global_step=step) except tf.errors.OutOfRangeError: break return total_loss, batches def validation(): f1 = 0 samples = 0 total_validation_loss = 0 total_time = 0 sess.run(validation_iterator_initializer) iteration = 0 while True: try: t1 = time.time() loss, actual_batch_size, lab, res = validation_step() f1 += np.sum( list( map( lambda x: f1_score( x[0], x[1], average="macro"), zip(lab.tolist(), res.tolist())))) samples += actual_batch_size iteration += 1 total_validation_loss += loss delta_t = time.time() - t1 total_time += delta_t print("当前f1为:{}, loss 为{}, 花费{}秒".format( f1 / samples, loss, delta_t)) except tf.errors.OutOfRangeError: print("平均f1为:{}, 平均loss为{}, 总耗时{}秒".format( f1 / samples, total_validation_loss / iteration, total_time)) break return f1 / samples total_loss = 0 batches = 0 for i in range(FLAGS.num_epochs): sess.run(train_iterator_initializer) print("第{}个epoch".format(i)) total_loss, batches = train(total_loss, batches)
print(len(x_train)) print(x_train) print(x_train.shape) print(x_train.shape[0]) print(x_train.shape[1]) print(y_train) print(y_train.shape) print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") # 从text_cnn.py中导入进TextCNN类 cnn = TextCNN( # shape[0]就是读取矩阵第一维度的长度 # shape[1]就是读取矩阵第二维度的长度 sequence_length=x_train. shape[1], # x_train.shape[1]句子的个数,x_train.shape[0]样本的个数 # 单个句子的最大长度(1个句子中单词的个数) max_document_length num_classes=y_train.shape[1], # 分类的种类0,1,在这就是2 index=x_train, embedding_size=FLAGS.embedding_dim, # 向量化的维度128 filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), # 卷积层的层数3 num_filters=FLAGS.num_filters, # 卷积核个数128 l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) # 全局步骤 optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
batch_size = 32 embedding_dims = 50 epochs = 10 print('Loading data...') (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) print(len(x_train), 'train sequences') print(len(x_test), 'test sequences') print('Pad sequences (samples x time)...') x_train = sequence.pad_sequences(x_train, maxlen=maxlen) x_test = sequence.pad_sequences(x_test, maxlen=maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('Build model...') model = TextCNN(maxlen, max_features, embedding_dims).get_model() model.compile('adam', 'binary_crossentropy', metrics=['accuracy']) print('Train...') early_stopping = EarlyStopping(monitor='val_acc', patience=3, mode='max') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=[early_stopping], validation_data=(x_test, y_test)) print('Test...') result = model.predict(x_test)
dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) x_train, x_dev = x[:dev_sample_index], x[dev_sample_index:] y_train, y_dev = y[:dev_sample_index], y[dev_sample_index:] print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) print("x_train shape" + str(x_train.shape)) print("y_train shape" + str(y_train.shape)) #Training with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1]) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.cross_entropy) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar(
# Configure Training # ==================================================================== with tf.Graph().as_default(): # set session conf session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): ### 1. Make CNN model cnn = TextCNN(max_sentence_length=x_train.shape[1], num_hidden_nodes=list( map(int, FLAGS.num_hidden_nodes.split(","))), num_classes=y_train.shape[1], vocab_size=len(vocab_processor.vocabulary_), word_vector_length=FLAGS.word_vector_length, filter_heights=list( map(int, FLAGS.filter_heights.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda) ### 2. Define Traning procedure """ set optimizer, calculate gradient """ global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) grads_and_vars = optimizer.compute_gradients(cnn.loss) # grads_and_vars have gradient when each value is changed. # value mean (trainable == true) train_op = optimizer.apply_gradients(grads_and_vars,