def evaluate(eval_data, config):
""" Build evaluation graph and run. """
with tf.Graph().as_default():
with tf.variable_scope('cnn'):
if config.has_key('contextwise') and config['contextwise']:
import cnn_context
m = cnn_context.Model(config, is_train=False)
else:
import cnn
m = cnn.Model(config, is_train=False)
saver = tf.train.Saver(tf.all_variables())
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(config['train_dir'])
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise IOError("Loading checkpoint file failed!")
#embeddings = sess.run(tf.all_variables())[0]
print "\nStart evaluation\n"
#losses = []
#precision = []
#recall = []
#batches = util.batch_iter(eval_data, batch_size=config['batch_size'], num_epochs=1, shuffle=False)
#for batch in batches:
if config.has_key('contextwise') and config['contextwise']:
left_batch, middle_batch, right_batch, y_batch, _ = zip(
*eval_data)
feed = {
m.left: np.array(left_batch),
m.middle: np.array(middle_batch),
m.right: np.array(right_batch),
m.labels: np.array(y_batch)
}
else:
x_batch, y_batch, _ = zip(*eval_data)
feed = {
m.inputs: np.array(x_batch),
m.labels: np.array(y_batch)
}
loss, eval = sess.run([m.total_loss, m.eval_op], feed_dict=feed)
#losses.append(loss)
pre, rec = zip(*eval)
#precision.append(pre)
#recall.append(rec)
avg_precision = np.mean(np.array(pre))
avg_recall = np.mean(np.array(rec))
auc = util.calc_auc_pr(pre, rec)
f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])
print '%s: loss = %.6f, f1 = %.4f, auc = %.4f' % (datetime.now(),
loss, f1, auc)
return pre, rec
def label(eval_data, config):
""" Build evaluation graph and run. """
with tf.Graph().as_default():
with tf.variable_scope('cnn'):
if config.has_key('contextwise') and config['contextwise']:
import cnn_context
m = cnn_context.Model(config, is_train=False)
else:
import cnn
m = cnn.Model(config, is_train=False)
saver = tf.train.Saver(tf.global_variables())
tf_config = tf.ConfigProto()
if config.get("gpu_percentage", 0) > 0:
tf_config.gpu_options.per_process_gpu_memory_fraction = config.get(
"gpu_percentage", 0)
else:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
with tf.Session(config=tf_config) as sess:
ckpt = tf.train.get_checkpoint_state(config['train_dir'])
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise IOError("Loading checkpoint file failed!")
print "\nStart evaluation\n"
x_data = np.array(eval_data)
data_size = x_data.shape[0]
batch_size = 10
actual_output = []
start_i = 0
while start_i < data_size:
end_i = min(start_i + batch_size, data_size)
if config.has_key('contextwise') and config['contextwise']:
raise NotImplementedError
# left_batch, middle_batch, right_batch, y_batch, _ = zip(*eval_data)
# feed = {m.left: np.array(left_batch),
# m.middle: np.array(middle_batch),
# m.right: np.array(right_batch),
# m.labels: np.array(y_batch)}
else:
x_batch = eval_data[start_i:end_i]
feed = {m.inputs: x_batch}
current_actual_output, = sess.run([m.scores], feed_dict=feed)
actual_output.append(current_actual_output)
start_i = end_i
actual_output = np.concatenate(actual_output, axis=0)
return x_data, actual_output
def evaluate(eval_data, config):
""" Build evaluation graph and run. """
with tf.Graph().as_default():
with tf.variable_scope('cnn'):
if config.has_key('contextwise') and config['contextwise']:
import cnn_context
m = cnn_context.Model(config, is_train=False)
else:
import cnn
m = cnn.Model(config, is_train=False)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(config['train_dir'])
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise IOError("Loading checkpoint file failed!")
print "\nStart evaluation on test set ...\n"
if config.has_key('contextwise') and config['contextwise']:
left_batch, middle_batch, right_batch, y_batch, _ = zip(
*eval_data)
feed = {
m.left: np.array(left_batch),
m.middle: np.array(middle_batch),
m.right: np.array(right_batch),
m.labels: np.array(y_batch)
}
else:
x_batch, y_batch, _ = zip(*eval_data)
feed = {
m.inputs: np.array(x_batch),
m.labels: np.array(y_batch)
}
loss, eval = sess.run([m.total_loss, m.eval_op], feed_dict=feed)
pre, rec = zip(*eval)
auc = util.calc_auc_pr(pre, rec)
f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])
print '%s: loss = %.6f, p = %.4f, r = %4.4f, f1 = %.4f, auc = %.4f' % (
datetime.now(), loss, pre[5], rec[5], f1, auc)
return pre, rec
def train(train_data, test_data):
# train_dir
timestamp = str(int(time.time()))
out_dir = os.path.abspath(os.path.join(FLAGS.train_dir, timestamp))
# save flags
if not os.path.exists(out_dir):
os.mkdir(out_dir)
FLAGS._parse_flags()
config = dict(FLAGS.__flags.items())
# Window_size must not be larger than the sent_len
if config['sent_len'] < config['max_window']:
config['max_window'] = config['sent_len']
# flag to restore the contextwise model
config['contextwise'] = True
# save flags
util.dump_to_file(os.path.join(out_dir, 'flags.cPickle'), config)
print "Parameters:"
for k, v in config.iteritems():
print '%20s %r' % (k, v)
# max number of steps
num_batches_per_epoch = int(
np.ceil(float(len(train_data)) / FLAGS.batch_size))
max_steps = num_batches_per_epoch * FLAGS.num_epochs
with tf.Graph().as_default():
with tf.variable_scope('cnn', reuse=None):
m = cnn_context.Model(config, is_train=True)
with tf.variable_scope('cnn', reuse=True):
mtest = cnn_context.Model(config, is_train=False)
# checkpoint
saver = tf.train.Saver(tf.global_variables())
save_path = os.path.join(out_dir, 'model.ckpt')
summary_op = tf.summary.merge_all()
# session
with tf.Session().as_default() as sess:
proj_config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig(
)
embedding_left = proj_config.embeddings.add()
embedding_middle = proj_config.embeddings.add()
embedding_right = proj_config.embeddings.add()
embedding_left.tensor_name = m.W_emb_left.name
embedding_middle.tensor_name = m.W_emb_middle.name
embedding_right.tensor_name = m.W_emb_right.name
embedding_left.metadata_path = os.path.join(
FLAGS.data_dir, 'vocab.txt')
embedding_middle.metadata_path = os.path.join(
FLAGS.data_dir, 'vocab.txt')
embedding_right.metadata_path = os.path.join(
FLAGS.data_dir, 'vocab.txt')
train_summary_writer = tf.summary.FileWriter(os.path.join(
out_dir, "train"),
graph=sess.graph)
dev_summary_writer = tf.summary.FileWriter(os.path.join(
out_dir, "dev"),
graph=sess.graph)
tf.contrib.tensorboard.plugins.projector.visualize_embeddings(
train_summary_writer, proj_config)
tf.contrib.tensorboard.plugins.projector.visualize_embeddings(
dev_summary_writer, proj_config)
sess.run(tf.global_variables_initializer())
# assign pretrained embeddings
if FLAGS.use_pretrain:
print "Initialize model with pretrained embeddings..."
pretrained_embedding = np.load(
os.path.join(FLAGS.data_dir, 'emb.npy'))
m.assign_embedding(sess, pretrained_embedding)
# initialize parameters
current_lr = FLAGS.init_lr
lowest_loss_value = float("inf")
decay_step_counter = 0
global_step = 0
# evaluate on dev set
def dev_step(mtest, sess):
dev_loss = []
dev_auc = []
dev_f1_score = []
# create batch
test_batches = util.batch_iter(test_data,
batch_size=FLAGS.batch_size,
num_epochs=1,
shuffle=False)
for batch in test_batches:
left_batch, middle_batch, right_batch, y_batch, _ = zip(
*batch)
feed = {
mtest.left: np.array(left_batch),
mtest.middle: np.array(middle_batch),
mtest.right: np.array(right_batch),
mtest.labels: np.array(y_batch)
}
loss_value, eval_value = sess.run(
[mtest.total_loss, mtest.eval_op], feed_dict=feed)
dev_loss.append(loss_value)
pre, rec = zip(*eval_value)
dev_auc.append(util.calc_auc_pr(pre, rec))
dev_f1_score.append((2.0 * pre[5] * rec[5]) /
(pre[5] + rec[5])) # threshold = 0.5
return np.mean(dev_loss), np.mean(dev_auc), np.mean(
dev_f1_score)
# train loop
print "\nStart training (save checkpoints in %s)\n" % out_dir
train_loss = []
train_auc = []
train_f1_score = []
train_batches = util.batch_iter(train_data,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
for batch in train_batches:
batch_size = len(batch)
m.assign_lr(sess, current_lr)
global_step += 1
left_batch, middle_batch, right_batch, y_batch, a_batch = zip(
*batch)
feed = {
m.left: np.array(left_batch),
m.middle: np.array(middle_batch),
m.right: np.array(right_batch),
m.labels: np.array(y_batch)
}
if FLAGS.attention:
feed[m.attention] = np.array(a_batch)
start_time = time.time()
_, loss_value, eval_value = sess.run(
[m.train_op, m.total_loss, m.eval_op], feed_dict=feed)
proc_duration = time.time() - start_time
train_loss.append(loss_value)
pre, rec = zip(*eval_value)
auc = util.calc_auc_pr(pre, rec)
f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5]
) # threshold = 0.5
train_auc.append(auc)
train_f1_score.append(f1)
assert not np.isnan(loss_value), "Model loss is NaN."
# print log
if global_step % FLAGS.log_step == 0:
examples_per_sec = batch_size / proc_duration
format_str = '%s: step %d/%d, f1 = %.4f, auc = %.4f, loss = %.4f ' + \
'(%.1f examples/sec; %.3f sec/batch), lr: %.6f'
print format_str % (datetime.now(), global_step, max_steps,
f1, auc, loss_value, examples_per_sec,
proc_duration, current_lr)
# write summary
if global_step % FLAGS.summary_step == 0:
summary_str = sess.run(summary_op)
train_summary_writer.add_summary(summary_str, global_step)
dev_summary_writer.add_summary(summary_str, global_step)
# summary loss, f1
train_summary_writer.add_summary(_summary_for_scalar(
'loss', np.mean(train_loss)),
global_step=global_step)
train_summary_writer.add_summary(_summary_for_scalar(
'auc', np.mean(train_auc)),
global_step=global_step)
train_summary_writer.add_summary(_summary_for_scalar(
'f1', np.mean(train_f1_score)),
global_step=global_step)
dev_loss, dev_auc, dev_f1 = dev_step(mtest, sess)
dev_summary_writer.add_summary(_summary_for_scalar(
'loss', dev_loss),
global_step=global_step)
dev_summary_writer.add_summary(_summary_for_scalar(
'auc', dev_auc),
global_step=global_step)
dev_summary_writer.add_summary(_summary_for_scalar(
'f1', dev_f1),
global_step=global_step)
print "\n===== write summary ====="
print "%s: step %d/%d: train_loss = %.6f, train_auc = %.4f train_f1 = %.4f" \
% (datetime.now(), global_step, max_steps,
np.mean(train_loss), np.mean(train_auc), np.mean(train_f1_score))
print "%s: step %d/%d: dev_loss = %.6f, dev_auc = %.4f dev_f1 = %.4f\n" \
% (datetime.now(), global_step, max_steps, dev_loss, dev_auc, dev_f1)
# reset container
train_loss = []
train_auc = []
train_f1_score = []
# decay learning rate if necessary
if loss_value < lowest_loss_value:
lowest_loss_value = loss_value
decay_step_counter = 0
else:
decay_step_counter += 1
if decay_step_counter >= FLAGS.tolerance_step:
current_lr *= FLAGS.lr_decay
print '%s: step %d/%d, Learning rate decays to %.5f' % \
(datetime.now(), global_step, max_steps, current_lr)
decay_step_counter = 0
# stop learning if learning rate is too low
if current_lr < 1e-5:
break
# save checkpoint
if global_step % FLAGS.checkpoint_step == 0:
saver.save(sess, save_path, global_step=global_step)
saver.save(sess, save_path, global_step=global_step)
def evaluate(eval_data, config):
""" Build evaluation graph and run. """
with tf.Graph().as_default():
with tf.variable_scope('cnn'):
if config.has_key('contextwise') and config['contextwise']:
import cnn_context
m = cnn_context.Model(config, is_train=False)
else:
import cnn
m = cnn.Model(config, is_train=False)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(config['train_dir'])
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise IOError("Loading checkpoint file failed!")
#embeddings = sess.run(tf.global_variables())[0]
print "\nStart evaluation\n"
#losses = []
#precision = []
#recall = []
#batches = util.batch_iter(eval_data, batch_size=config['batch_size'], num_epochs=1, shuffle=False)
#for batch in batches:
if config.has_key('contextwise') and config['contextwise']:
left_batch, middle_batch, right_batch, y_batch, _ = zip(*eval_data)
feed = {m.left: np.array(left_batch),
m.middle: np.array(middle_batch),
m.right: np.array(right_batch),
m.labels: np.array(y_batch)}
else:
x_batch, y_batch, _ = zip(*eval_data)
feed = {m.inputs: np.array(x_batch), m.labels: np.array(y_batch)}
loss, eval, actual_output, eval_per_class = sess.run([m.total_loss, m.eval_op, m.scores, m.eval_class_op], feed_dict=feed)
#losses.append(loss)
pre, rec = zip(*eval)
#precision.append(pre)
#recall.append(rec)
avg_precision = np.mean(np.array(pre))
avg_recall = np.mean(np.array(rec))
auc = util.calc_auc_pr(pre, rec)
f1 = (2.0 * pre[5] * rec[5]) / (pre[5] + rec[5])
print '%s: Overall\nloss = %.6f, f1 = %.4f, auc = %.4f' % (datetime.now(), loss, f1, auc)
pre_per_class, rec_per_class = zip(*eval_per_class)
num_class = len(pre_per_class)
for class_i in range(num_class):
current_pre = pre_per_class[class_i]
current_rec = rec_per_class[class_i]
current_auc = util.calc_auc_pr(current_pre, current_rec)
current_f1 = (2.0 * current_pre[5] * current_rec[5]) / (current_pre[5] + current_rec[5])
print 'Class "%s": precision = %.4f, recall = %.4f, f1 = %.4f, auc = %.4f' % (CLASS_NAMES[class_i], current_pre[5], current_rec[5], current_f1, current_auc)
x_batch = np.array(x_batch)
y_batch = np.array(y_batch)
# Now calculate the true probability distribution using softmax.
actual_output_exp = np.exp(actual_output)
actual_output_softmax = actual_output_exp / np.sum(actual_output_exp, axis=1, keepdims=True)
plot_precision_recall(y_batch,actual_output_softmax)
return pre, rec, x_batch, y_batch, actual_output