def report(y_true, y_pred, class_names, threshold=0.5): tp = np.logical_and(y_true, y_pred) fp = np.logical_and(np.logical_not(y_true), y_pred) fn = np.logical_and(y_true, np.logical_not(y_pred)) pre = np.sum(tp.astype(float), axis=1)/np.sum(np.logical_or(tp, fp).astype(float), axis=1) rec = np.sum(tp.astype(float), axis=1)/np.sum(np.logical_or(tp, fn).astype(float), axis=1) f1 = (2.0 * pre * rec)/(pre + rec) count = np.sum(y_true, axis=0) auc_array = [] ret = '%45s\t P\t R\t F1\t AUC\t C\n' % ' ' ret += '\t' + '-' * 80 + '\n' for i, (c, p, r, f, s) in enumerate(zip(class_names, pre[5], rec[5], f1[5], count)): auc = util.calc_auc_pr(pre[:, i], rec[:, i]) auc_array.append(auc) ret += '%45s\t%.4f\t%.4f\t%.4f\t%.4f\t%4d\n' % (c, p, r, f, auc, s) ret += '\t' + '-' * 80 + '\n' auc = np.array(auc_array) try: idx = list(np.linspace(0, 1, 11)).index(threshold) except ValueError: idx = 5 p = pre[idx, np.isfinite(pre[idx])]*count[np.isfinite(pre[idx])] r = rec[idx, np.isfinite(rec[idx])]*count[np.isfinite(rec[idx])] f = f1[idx, np.isfinite(f1[idx])]*count[np.isfinite(f1[idx])] a = auc[np.isfinite(auc)]*count[np.isfinite(auc)] ret += '%45s\t%.4f\t%.4f\t%.4f\t%.4f\t%4d\n' % ('avg.', np.sum(p)/count.sum(dtype=float), np.sum(r)/count.sum(dtype=float), np.sum(f)/count.sum(dtype=float), np.sum(a)/count.sum(dtype=float), count.sum()) return ret
def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(test_data, batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: left_batch, right_batch, y_batch, n_batch = zip(*batch) feed = { mtest.left: np.array(left_batch), mtest.right: np.array(right_batch), mtest.labels: np.array(y_batch) } if FLAGS.negative: feed[mtest.negative] = np.array(n_batch) loss_value, eval_value = sess.run( [mtest.total_loss, mtest.eval_op], feed_dict=feed) dev_loss.append(loss_value) pre, rec = zip(*eval_value) dev_auc.append(util.calc_auc_pr(pre, rec)) dev_f1_score.append((2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])) # threshold = 0.5 return (np.mean(dev_loss), np.mean(dev_auc), np.mean(dev_f1_score))
def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(test_data, batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: x_batch, y_batch, _ = zip(*batch) loss_value, eval_value = sess.run( [mtest.total_loss, mtest.eval_op], feed_dict={ mtest.inputs: np.array(x_batch), mtest.labels: np.array(y_batch) }) dev_loss.append(loss_value) pre, rec = zip(*eval_value) # look at the 5th index, which corresponds to a threshold = 0.5 threshold = 5 dev_auc.append(util.calc_auc_pr(pre, rec, threshold)) dev_f1_score.append( (2.0 * pre[threshold] * rec[threshold]) / (pre[threshold] + rec[threshold])) return np.mean(dev_loss), np.mean(dev_auc), np.mean( dev_f1_score)
def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(test_data, batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: x_batch, y_batch, _ = zip(*batch) #a_batch = np.ones((len(batch), 1), dtype=np.float32) / len(batch) # average loss_value, eval_value = sess.run( [mtest.total_loss, mtest.eval_op], feed_dict={ mtest.inputs: np.array(x_batch), mtest.labels: np.array(y_batch) }) dev_loss.append(loss_value) pre, rec = zip(*eval_value) dev_auc.append(util.calc_auc_pr(pre, rec)) dev_f1_score.append((2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])) # threshold = 0.5 return (np.mean(dev_loss), np.mean(dev_auc), np.mean(dev_f1_score))
def evaluate(eval_data, config): """ Build evaluation graph and run. """ with tf.Graph().as_default(): with tf.variable_scope('cnn'): if config.has_key('contextwise') and config['contextwise']: import cnn_context m = cnn_context.Model(config, is_train=False) else: import cnn m = cnn.Model(config, is_train=False) saver = tf.train.Saver(tf.all_variables()) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(config['train_dir']) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise IOError("Loading checkpoint file failed!") #embeddings = sess.run(tf.all_variables())[0] print "\nStart evaluation\n" #losses = [] #precision = [] #recall = [] #batches = util.batch_iter(eval_data, batch_size=config['batch_size'], num_epochs=1, shuffle=False) #for batch in batches: if config.has_key('contextwise') and config['contextwise']: left_batch, middle_batch, right_batch, y_batch, _ = zip( *eval_data) feed = { m.left: np.array(left_batch), m.middle: np.array(middle_batch), m.right: np.array(right_batch), m.labels: np.array(y_batch) } else: x_batch, y_batch, _ = zip(*eval_data) feed = { m.inputs: np.array(x_batch), m.labels: np.array(y_batch) } loss, eval = sess.run([m.total_loss, m.eval_op], feed_dict=feed) #losses.append(loss) pre, rec = zip(*eval) #precision.append(pre) #recall.append(rec) avg_precision = np.mean(np.array(pre)) avg_recall = np.mean(np.array(rec)) auc = util.calc_auc_pr(pre, rec) f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5]) print '%s: loss = %.6f, f1 = %.4f, auc = %.4f' % (datetime.now(), loss, f1, auc) return pre, rec
def evaluate(eval_data, config): """ Build evaluation graph and run. """ with tf.Graph().as_default(): with tf.variable_scope('cnn'): if config.has_key('contextwise') and config['contextwise']: import cnn_context m = cnn_context.Model(config, is_train=False) else: import cnn m = cnn.Model(config, is_train=False) saver = tf.train.Saver(tf.global_variables()) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(config['train_dir']) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise IOError("Loading checkpoint file failed!") print "\nStart evaluation on test set ...\n" if config.has_key('contextwise') and config['contextwise']: left_batch, middle_batch, right_batch, y_batch, _ = zip( *eval_data) feed = { m.left: np.array(left_batch), m.middle: np.array(middle_batch), m.right: np.array(right_batch), m.labels: np.array(y_batch) } else: x_batch, y_batch, _ = zip(*eval_data) feed = { m.inputs: np.array(x_batch), m.labels: np.array(y_batch) } loss, eval = sess.run([m.total_loss, m.eval_op], feed_dict=feed) pre, rec = zip(*eval) auc = util.calc_auc_pr(pre, rec) f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5]) print '%s: loss = %.6f, p = %.4f, r = %4.4f, f1 = %.4f, auc = %.4f' % ( datetime.now(), loss, pre[5], rec[5], f1, auc) return pre, rec
def train(train_data, test_data): # train_dir timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(FLAGS.train_dir, timestamp)) # save flags if not os.path.exists(out_dir): os.mkdir(out_dir) FLAGS._parse_flags() config = dict(FLAGS.__flags.items()) # Window_size must not be larger than the sent_len if config['sent_len'] < config['max_window']: config['max_window'] = config['sent_len'] # flag to restore the contextwise model config['split'] = True # save flags config['train_dir'] = out_dir util.dump_to_file(os.path.join(out_dir, 'flags.cPickle'), config) # display parameter settings print 'Parameters:' for k, v in config.iteritems(): print '\t' + k + '=' + str(v) # max number of steps num_batches_per_epoch = int( np.ceil(float(len(train_data)) / FLAGS.batch_size)) max_steps = num_batches_per_epoch * FLAGS.num_epochs with tf.Graph().as_default(): with tf.variable_scope('cnn', reuse=None): m = cnn_split.Model(config, is_train=True) with tf.variable_scope('cnn', reuse=True): mtest = cnn_split.Model(config, is_train=False) # checkpoint saver = tf.train.Saver(tf.all_variables()) save_path = os.path.join(out_dir, 'model.ckpt') summary_op = tf.merge_all_summaries() # session sess = tf.Session(config=tf.ConfigProto( log_device_placement=FLAGS.log_device_placement)) with sess.as_default(): train_summary_writer = tf.train.SummaryWriter(os.path.join( out_dir, "train"), graph=sess.graph) dev_summary_writer = tf.train.SummaryWriter(os.path.join( out_dir, "dev"), graph=sess.graph) sess.run(tf.initialize_all_variables()) # assign pretrained embeddings if FLAGS.use_pretrain: print "Initializing model with pretrained embeddings ..." pretrained_embedding = np.load( os.path.join(FLAGS.data_dir, 'emb.npy')) m.assign_embedding(sess, pretrained_embedding) # initialize parameters current_lr = FLAGS.init_lr lowest_loss_value = float("inf") decay_step_counter = 0 global_step = 0 # evaluate on dev set def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(test_data, batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: left_batch, right_batch, y_batch, n_batch = zip(*batch) feed = { mtest.left: np.array(left_batch), mtest.right: np.array(right_batch), mtest.labels: np.array(y_batch) } if FLAGS.negative: feed[mtest.negative] = np.array(n_batch) loss_value, eval_value = sess.run( [mtest.total_loss, mtest.eval_op], feed_dict=feed) dev_loss.append(loss_value) pre, rec = zip(*eval_value) dev_auc.append(util.calc_auc_pr(pre, rec)) dev_f1_score.append((2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])) # threshold = 0.5 return (np.mean(dev_loss), np.mean(dev_auc), np.mean(dev_f1_score)) # train loop print "\nStart training (save checkpoints in %s)\n" % out_dir train_loss = [] train_auc = [] train_f1_score = [] train_batches = util.batch_iter(train_data, batch_size=FLAGS.batch_size, num_epochs=FLAGS.num_epochs) for batch in train_batches: batch_size = len(batch) m.assign_lr(sess, current_lr) global_step += 1 left_batch, right_batch, y_batch, n_batch = zip(*batch) feed = { m.left: np.array(left_batch), m.right: np.array(right_batch), m.labels: np.array(y_batch) } if FLAGS.negative: feed[m.negative] = np.array(n_batch) start_time = time.time() _, loss_value, eval_value = sess.run( [m.train_op, m.total_loss, m.eval_op], feed_dict=feed) proc_duration = time.time() - start_time train_loss.append(loss_value) pre, rec = zip(*eval_value) auc = util.calc_auc_pr(pre, rec) f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5] ) # threshold = 0.5 train_auc.append(auc) train_f1_score.append(f1) assert not np.isnan(loss_value), "Model loss is NaN." # print log if global_step % FLAGS.log_step == 0: examples_per_sec = batch_size / proc_duration format_str = '%s: step %d/%d, f1 = %.4f, auc = %.4f, loss = %.4f ' + \ '(%.1f examples/sec; %.3f sec/batch), lr: %.6f' print format_str % (datetime.now(), global_step, max_steps, f1, auc, loss_value, examples_per_sec, proc_duration, current_lr) # write summary if global_step % FLAGS.summary_step == 0: summary_str = sess.run(summary_op) train_summary_writer.add_summary(summary_str, global_step) dev_summary_writer.add_summary(summary_str, global_step) # summary loss, f1 train_summary_writer.add_summary(_summary_for_scalar( 'loss', np.mean(train_loss)), global_step=global_step) train_summary_writer.add_summary(_summary_for_scalar( 'auc', np.mean(train_auc)), global_step=global_step) train_summary_writer.add_summary(_summary_for_scalar( 'f1', np.mean(train_f1_score)), global_step=global_step) dev_loss, dev_auc, dev_f1 = dev_step(mtest, sess) dev_summary_writer.add_summary(_summary_for_scalar( 'loss', dev_loss), global_step=global_step) dev_summary_writer.add_summary(_summary_for_scalar( 'auc', dev_auc), global_step=global_step) dev_summary_writer.add_summary(_summary_for_scalar( 'f1', dev_f1), global_step=global_step) print "\n===== write summary =====" print "%s: step %d/%d: train_loss = %.6f, train_auc = %.4f train_f1 = %.4f" \ % (datetime.now(), global_step, max_steps, np.mean(train_loss), np.mean(train_auc), np.mean(train_f1_score)) print "%s: step %d/%d: dev_loss = %.6f, dev_auc = %.4f dev_f1 = %.4f\n" \ % (datetime.now(), global_step, max_steps, dev_loss, dev_auc, dev_f1) # reset container train_loss = [] train_auc = [] train_f1_score = [] # decay learning rate if necessary if loss_value < lowest_loss_value: lowest_loss_value = loss_value decay_step_counter = 0 else: decay_step_counter += 1 if decay_step_counter >= FLAGS.tolerance_step: current_lr *= FLAGS.lr_decay print '%s: step %d/%d, Learning rate decays to %.5f' % \ (datetime.now(), global_step, max_steps, current_lr) decay_step_counter = 0 # stop learning if learning rate is too low if current_lr < 1e-5: break # save checkpoint if global_step % FLAGS.checkpoint_step == 0: saver.save(sess, save_path, global_step=global_step) saver.save(sess, save_path, global_step=global_step)
def train(train_data, test_data): # train_dir timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(FLAGS.train_dir, timestamp)) # save flags if not os.path.exists(out_dir): os.mkdir(out_dir) FLAGS._parse_flags() config = dict(FLAGS.__flags.items()) # Window_size must not be larger than the sent_len if config['sent_len'] < config['max_window']: config['max_window'] = config['sent_len'] util.dump_to_file(os.path.join(out_dir, 'flags.cPickle'), config) print("Parameters:") for k, v in config.items(): print('%20s %r' % (k, v)) num_batches_per_epoch = int( np.ceil(float(len(train_data)) / FLAGS.batch_size)) max_steps = num_batches_per_epoch * FLAGS.num_epochs with tf.Graph().as_default(): with tf.variable_scope('cnn', reuse=None): m = cnn.Model(config, is_train=True) with tf.variable_scope('cnn', reuse=True): mtest = cnn.Model(config, is_train=False) # checkpoint saver = tf.train.Saver(tf.global_variables()) save_path = os.path.join(out_dir, 'model.ckpt') summary_op = tf.summary.merge_all() # session with tf.Session().as_default() as sess: proj_config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig( ) embedding = proj_config.embeddings.add() embedding.tensor_name = m.W_emb.name embedding.metadata_path = os.path.join(FLAGS.data_dir, 'vocab.txt') train_summary_writer = tf.summary.FileWriter(os.path.join( out_dir, "train"), graph=sess.graph) dev_summary_writer = tf.summary.FileWriter(os.path.join( out_dir, "dev"), graph=sess.graph) tf.contrib.tensorboard.plugins.projector.visualize_embeddings( train_summary_writer, proj_config) tf.contrib.tensorboard.plugins.projector.visualize_embeddings( dev_summary_writer, proj_config) sess.run(tf.global_variables_initializer()) # assign pretrained embeddings if FLAGS.use_pretrain: print("Initialize model with pretrained embeddings...") pretrained_embedding = np.load( os.path.join(FLAGS.data_dir, 'emb.npy')) m.assign_embedding(sess, pretrained_embedding) # initialize parameters current_lr = FLAGS.init_lr lowest_loss_value = float("inf") decay_step_counter = 0 global_step = 0 # evaluate on dev set def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(test_data, batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: x_batch, y_batch, _ = zip(*batch) loss_value, eval_value = sess.run( [mtest.total_loss, mtest.eval_op], feed_dict={ mtest.inputs: np.array(x_batch), mtest.labels: np.array(y_batch) }) dev_loss.append(loss_value) pre, rec = zip(*eval_value) # look at the 5th index, which corresponds to a threshold = 0.5 threshold = 5 dev_auc.append(util.calc_auc_pr(pre, rec, threshold)) dev_f1_score.append( (2.0 * pre[threshold] * rec[threshold]) / (pre[threshold] + rec[threshold])) return np.mean(dev_loss), np.mean(dev_auc), np.mean( dev_f1_score) # train loop print("\nStart training (save checkpoints in %s)\n" % out_dir) train_loss = [] train_auc = [] train_f1_score = [] train_batches = util.batch_iter(train_data, batch_size=FLAGS.batch_size, num_epochs=FLAGS.num_epochs) for batch in train_batches: batch_size = len(batch) m.assign_lr(sess, current_lr) global_step += 1 x_batch, y_batch, a_batch = zip(*batch) feed = { m.inputs: np.array(x_batch), m.labels: np.array(y_batch) } if FLAGS.attention: feed[m.attention] = np.array(a_batch) start_time = time.time() _, loss_value, eval_value = sess.run( [m.train_op, m.total_loss, m.eval_op], feed_dict=feed) proc_duration = time.time() - start_time train_loss.append(loss_value) pre, rec = zip(*eval_value) # look at the 5th index, which corresponds to a threshold = 0.5 threshold = 5 auc = util.calc_auc_pr(pre, rec, threshold) f1 = (2.0 * pre[threshold] * rec[threshold]) / (pre[threshold] + rec[threshold]) train_auc.append(auc) train_f1_score.append(f1) assert not np.isnan(loss_value), "Model loss is NaN." # print log if global_step % FLAGS.log_step == 0: examples_per_sec = batch_size / proc_duration format_str = '%s: step %d/%d, f1 = %.4f, auc = %.4f, loss = %.4f ' + \ '(%.1f examples/sec; %.3f sec/batch), lr: %.6f' print(format_str % (datetime.now(), global_step, max_steps, f1, auc, loss_value, examples_per_sec, proc_duration, current_lr)) # write summary if global_step % FLAGS.summary_step == 0: summary_str = sess.run(summary_op) train_summary_writer.add_summary(summary_str, global_step) dev_summary_writer.add_summary(summary_str, global_step) # summary loss, f1 train_summary_writer.add_summary(_summary_for_scalar( 'loss', np.mean(train_loss)), global_step=global_step) train_summary_writer.add_summary(_summary_for_scalar( 'auc', np.mean(train_auc)), global_step=global_step) train_summary_writer.add_summary(_summary_for_scalar( 'f1', np.mean(train_f1_score)), global_step=global_step) dev_loss, dev_auc, dev_f1 = dev_step(mtest, sess) dev_summary_writer.add_summary(_summary_for_scalar( 'loss', dev_loss), global_step=global_step) dev_summary_writer.add_summary(_summary_for_scalar( 'auc', dev_auc), global_step=global_step) dev_summary_writer.add_summary(_summary_for_scalar( 'f1', dev_f1), global_step=global_step) print("\n===== write summary =====") print("%s: step %d/%d: train_loss = %.6f, train_auc = %.4f, train_f1 = %.4f" \ % (datetime.now(), global_step, max_steps, np.mean(train_loss), np.mean(train_auc), np.mean(train_f1_score))) print("%s: step %d/%d: dev_loss = %.6f, dev_auc = %.4f, dev_f1 = %.4f\n" \ % (datetime.now(), global_step, max_steps, dev_loss, dev_auc, dev_f1)) # reset container train_loss = [] train_auc = [] train_f1_score = [] # decay learning rate if necessary if loss_value < lowest_loss_value: lowest_loss_value = loss_value decay_step_counter = 0 else: decay_step_counter += 1 if decay_step_counter >= FLAGS.tolerance_step: current_lr *= FLAGS.lr_decay print('%s: step %d/%d, Learning rate decays to %.5f' % \ (datetime.now(), global_step, max_steps, current_lr)) decay_step_counter = 0 # stop learning if learning rate is too low if current_lr < 1e-5: break # save checkpoint if global_step % FLAGS.checkpoint_step == 0: saver.save(sess, save_path, global_step=global_step) saver.save(sess, save_path, global_step=global_step)
def evaluate(eval_data, config): """ Build evaluation graph and run. """ with tf.Graph().as_default(): with tf.variable_scope('cnn'): if config.has_key('contextwise') and config['contextwise']: import cnn_context m = cnn_context.Model(config, is_train=False) else: import cnn m = cnn.Model(config, is_train=False) saver = tf.train.Saver(tf.global_variables()) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(config['train_dir']) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise IOError("Loading checkpoint file failed!") #embeddings = sess.run(tf.global_variables())[0] print "\nStart evaluation\n" #losses = [] #precision = [] #recall = [] #batches = util.batch_iter(eval_data, batch_size=config['batch_size'], num_epochs=1, shuffle=False) #for batch in batches: if config.has_key('contextwise') and config['contextwise']: left_batch, middle_batch, right_batch, y_batch, _ = zip(*eval_data) feed = {m.left: np.array(left_batch), m.middle: np.array(middle_batch), m.right: np.array(right_batch), m.labels: np.array(y_batch)} else: x_batch, y_batch, _ = zip(*eval_data) feed = {m.inputs: np.array(x_batch), m.labels: np.array(y_batch)} loss, eval, actual_output, eval_per_class = sess.run([m.total_loss, m.eval_op, m.scores, m.eval_class_op], feed_dict=feed) #losses.append(loss) pre, rec = zip(*eval) #precision.append(pre) #recall.append(rec) avg_precision = np.mean(np.array(pre)) avg_recall = np.mean(np.array(rec)) auc = util.calc_auc_pr(pre, rec) f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5]) print '%s: Overall\nloss = %.6f, f1 = %.4f, auc = %.4f' % (datetime.now(), loss, f1, auc) pre_per_class, rec_per_class = zip(*eval_per_class) num_class = len(pre_per_class) for class_i in range(num_class): current_pre = pre_per_class[class_i] current_rec = rec_per_class[class_i] current_auc = util.calc_auc_pr(current_pre, current_rec) current_f1 = (2.0 * current_pre[5] * current_rec[5]) / (current_pre[5] + current_rec[5]) print 'Class "%s": precision = %.4f, recall = %.4f, f1 = %.4f, auc = %.4f' % (CLASS_NAMES[class_i], current_pre[5], current_rec[5], current_f1, current_auc) x_batch = np.array(x_batch) y_batch = np.array(y_batch) # Now calculate the true probability distribution using softmax. actual_output_exp = np.exp(actual_output) actual_output_softmax = actual_output_exp / np.sum(actual_output_exp, axis=1, keepdims=True) plot_precision_recall(y_batch,actual_output_softmax) return pre, rec, x_batch, y_batch, actual_output
def train(train_data, test_data, FLAGS = tf.app.flags.FLAGS): # # train_dir # timestamp = str(int(time.time())) # out_dir = os.path.abspath(os.path.join(FLAGS.train_dir, timestamp)) # # # save flags # if not os.path.exists(out_dir): # os.mkdir(out_dir) # FLAGS._parse_flags() # config = dict(FLAGS.__flags.items()) # # # Window_size must not be larger than the sent_len # if config['sent_len'] < config['max_window']: # config['max_window'] = config['sent_len'] # # util.dump_to_file(os.path.join(out_dir, 'flags.cPickle'), config) train_x = get_key_phrases(train_data) _, train_y, _ = zip(*train_data) test_x = get_key_phrases(test_data) _, test_y, _ = zip(*test_data) # # assign pretrained embeddings # if FLAGS.use_pretrain: print "Initialize model with pretrained embeddings..." print("Please don't forget to change the vocab size to the corresponding on in the embedding.") pretrained_embedding = np.load(os.path.join(FLAGS.data_dir, 'emb.npy')) train_x = key_phrase_indices_to_embedding(train_x, pretrained_embedding) test_x = key_phrase_indices_to_embedding(test_x, pretrained_embedding) # Use SVM. But SVM does not output a probability # train_y = np.argmax(train_y, axis=1) # test_y = np.argmax(test_y, axis=1) # clf = svm.SVC(class_weight='balanced') # clf.fit(train_x, train_y) # predicted_test_y = clf.predict(test_x) # Use fully connected multilayer nn. config = dict(FLAGS.__flags.items()) # train_dir timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(FLAGS.train_dir, timestamp)) # save flags if not os.path.exists(out_dir): os.mkdir(out_dir) FLAGS._parse_flags() config = dict(FLAGS.__flags.items()) util.dump_to_file(os.path.join(out_dir, 'flags.cPickle'), config) num_batches_per_epoch = int(np.ceil(float(len(train_data))/FLAGS.batch_size)) max_steps = num_batches_per_epoch * FLAGS.num_epochs with tf.Graph().as_default(): with tf.variable_scope('multilayer', reuse=None): m = multilayer.Model(config, is_train=True) with tf.variable_scope('multilayer', reuse=True): mtest = multilayer.Model(config, is_train=False) # checkpoint saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) save_path = os.path.join(out_dir, 'model.ckpt') try: summary_op = tf.summary.merge_all() except: summary_op = tf.merge_all_summaries() # session config = tf.ConfigProto(log_device_placement=FLAGS.log_device_placement) if FLAGS.gpu_percentage > 0: config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_percentage else: config = tf.ConfigProto( log_device_placement=FLAGS.log_device_placement, device_count={'GPU': 0} ) sess = tf.Session(config=config) with sess.as_default(): train_summary_writer = tf.train.SummaryWriter(os.path.join(out_dir, "train"), graph=sess.graph) dev_summary_writer = tf.train.SummaryWriter(os.path.join(out_dir, "dev"), graph=sess.graph) try: sess.run(tf.global_variables_initializer()) except: sess.run(tf.initialize_all_variables()) # # assign pretrained embeddings # if FLAGS.use_pretrain: # print "Initialize model with pretrained embeddings..." # print("Please don't forget to change the vocab size to the corresponding on in the embedding.") # pretrained_embedding = np.load(os.path.join(FLAGS.data_dir, 'emb.npy')) # m.assign_embedding(sess, pretrained_embedding) # initialize parameters current_lr = FLAGS.init_lr lowest_loss_value = float("inf") decay_step_counter = 0 global_step = 0 # evaluate on dev set def dev_step(mtest, sess): dev_loss = [] dev_auc = [] dev_f1_score = [] # create batch test_batches = util.batch_iter(zip(test_x, test_y), batch_size=FLAGS.batch_size, num_epochs=1, shuffle=False) for batch in test_batches: x_batch, y_batch, = zip(*batch) # a_batch = np.ones((len(batch), 1), dtype=np.float32) / len(batch) # average loss_value, eval_value = sess.run([mtest.total_loss, mtest.eval_op], feed_dict={mtest.inputs: np.array(x_batch), mtest.labels: np.array(y_batch)}) dev_loss.append(loss_value) pre, rec = zip(*eval_value) dev_auc.append(util.calc_auc_pr(pre, rec)) dev_f1_score.append((2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])) # threshold = 0.5 return (np.mean(dev_loss), np.mean(dev_auc), np.mean(dev_f1_score)) # train loop print "\nStart training (save checkpoints in %s)\n" % out_dir train_loss = [] train_auc = [] train_f1_score = [] train_batches = util.batch_iter(zip(train_x, train_y), batch_size=FLAGS.batch_size, num_epochs=FLAGS.num_epochs) for batch in train_batches: batch_size = len(batch) m.assign_lr(sess, current_lr) global_step += 1 x_batch, y_batch, = zip(*batch) feed = {m.inputs: np.array(x_batch), m.labels: np.array(y_batch)} start_time = time.time() _, loss_value, eval_value = sess.run([m.train_op, m.total_loss, m.eval_op], feed_dict=feed) proc_duration = time.time() - start_time train_loss.append(loss_value) pre, rec = zip(*eval_value) auc = util.calc_auc_pr(pre, rec) f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5]) # threshold = 0.5 train_auc.append(auc) train_f1_score.append(f1) assert not np.isnan(loss_value), "Model loss is NaN." # print log if global_step % FLAGS.log_step == 0: examples_per_sec = batch_size / proc_duration format_str = '%s: step %d/%d, f1 = %.4f, auc = %.4f, loss = %.4f ' + \ '(%.1f examples/sec; %.3f sec/batch), lr: %.6f' print format_str % (datetime.now(), global_step, max_steps, f1, auc, loss_value, examples_per_sec, proc_duration, current_lr) # write summary if global_step % FLAGS.summary_step == 0: summary_str = sess.run(summary_op) train_summary_writer.add_summary(summary_str, global_step) dev_summary_writer.add_summary(summary_str, global_step) # summary loss, f1 train_summary_writer.add_summary( _summary_for_scalar('loss', np.mean(train_loss)), global_step=global_step) train_summary_writer.add_summary( _summary_for_scalar('auc', np.mean(train_auc)), global_step=global_step) train_summary_writer.add_summary( _summary_for_scalar('f1', np.mean(train_f1_score)), global_step=global_step) dev_loss, dev_auc, dev_f1 = dev_step(mtest, sess) dev_summary_writer.add_summary( _summary_for_scalar('loss', dev_loss), global_step=global_step) dev_summary_writer.add_summary( _summary_for_scalar('auc', dev_auc), global_step=global_step) dev_summary_writer.add_summary( _summary_for_scalar('f1', dev_f1), global_step=global_step) print "\n===== write summary =====" print "%s: step %d/%d: train_loss = %.6f, train_auc = %.4f, train_f1 = %.4f" \ % (datetime.now(), global_step, max_steps, np.mean(train_loss), np.mean(train_auc), np.mean(train_f1_score)) print "%s: step %d/%d: dev_loss = %.6f, dev_auc = %.4f, dev_f1 = %.4f\n" \ % (datetime.now(), global_step, max_steps, dev_loss, dev_auc, dev_f1) # reset container train_loss = [] train_auc = [] train_f1_score = [] # decay learning rate if necessary if loss_value < lowest_loss_value: lowest_loss_value = loss_value decay_step_counter = 0 else: decay_step_counter += 1 if decay_step_counter >= FLAGS.tolerance_step: current_lr *= FLAGS.lr_decay print '%s: step %d/%d, Learning rate decays to %.5f' % \ (datetime.now(), global_step, max_steps, current_lr) decay_step_counter = 0 # stop learning if learning rate is too low if current_lr < 1e-5: break # save checkpoint if global_step % FLAGS.checkpoint_step == 0: saver.save(sess, save_path, global_step=global_step) saver.save(sess, save_path, global_step=global_step) # Lastly evaluate the test set loss_value, predicted_test_y_logits = sess.run([mtest.total_loss, mtest.scores], feed_dict={mtest.inputs: np.array(test_x), mtest.labels: np.array(test_y)}) predicted_test_y = np.argmax(predicted_test_y_logits, axis=1) test_y = np.argmax(test_y, axis=1) result = (predicted_test_y == test_y) accuracy = np.sum(result.astype(np.int32)) / float(result.shape[0]) print("Overall %f%% answers were correct. " %(float(accuracy * 100))) epsilon = 0.00000001 num_categories = 3 true_positive_per_category = [np.bitwise_and(test_y==category_i, predicted_test_y==category_i) for category_i in range(num_categories)] false_positive_per_category = [np.bitwise_and(test_y!=category_i, predicted_test_y==category_i) for category_i in range(num_categories)] true_negative_per_category = [np.bitwise_and(test_y!=category_i, predicted_test_y!=category_i) for category_i in range(num_categories)] false_negative_per_category = [np.bitwise_and(test_y==category_i, predicted_test_y!=category_i) for category_i in range(num_categories)] precision_per_category = [np.sum(true_positive_per_category[category_i].astype(np.int32)) / float(np.sum(true_positive_per_category[category_i].astype(np.int32)) + np.sum(false_positive_per_category[category_i].astype(np.int32)) + epsilon) for category_i in range(num_categories)] recall_per_category = [np.sum(true_positive_per_category[category_i].astype(np.int32)) / float(np.sum(true_positive_per_category[category_i].astype(np.int32)) + np.sum(false_negative_per_category[category_i].astype(np.int32)) + epsilon) for category_i in range(num_categories)] f1_per_category = [2 / (1 / (precision_per_category[category_i] + epsilon) + 1 / (recall_per_category[category_i] + epsilon)) for category_i in range(num_categories)] for category_i in range(num_categories): print("Category %d has f1 score: %f, precision: %f, and recall %f" %(category_i, f1_per_category[category_i], precision_per_category[category_i], recall_per_category[category_i])) return test_x, predicted_test_y_logits