def train_sann():
"""Training RNN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load word2vec model
word2idx, embedding_matrix = dh.load_word2vec_matrix(args.word2vec_file)
# Load sentences, labels, and training parameters
logger.info("Loading data...")
logger.info("Data processing...")
train_data = dh.load_data_and_labels(args, args.train_file, word2idx)
val_data = dh.load_data_and_labels(args, args.validation_file, word2idx)
# Build a graph and sann object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
sann = TextSANN(sequence_length=args.pad_seq_len,
vocab_size=len(word2idx),
embedding_type=args.embedding_type,
embedding_size=args.embedding_dim,
lstm_hidden_size=args.lstm_dim,
attention_unit_size=args.attention_dim,
attention_hops_size=args.attention_hops_dim,
fc_hidden_size=args.fc_dim,
num_classes=args.num_classes,
l2_reg_lambda=args.l2_lambda,
pretrained_embedding=embedding_matrix)
# Define training procedure
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(
learning_rate=args.learning_rate,
global_step=sann.global_step,
decay_steps=args.decay_steps,
decay_rate=args.decay_rate,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(sann.loss))
grads, _ = tf.clip_by_global_norm(grads,
clip_norm=args.norm_ratio)
train_op = optimizer.apply_gradients(
zip(grads, vars),
global_step=sann.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
out_dir = dh.get_out_dir(OPTION, logger)
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", sann.loss)
# Train summaries
train_summary_op = tf.summary.merge(
[loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=args.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir,
num_to_keep=3,
maximize=True)
if OPTION == 'R':
# Load sann model
logger.info("Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
if OPTION == 'T':
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = args.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer,
config)
# Save the embedding visualization
saver.save(
sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(sann.global_step)
def train_step(batch_data):
"""A single training step."""
x_f, x_b, y_onehot = zip(*batch_data)
feed_dict = {
sann.input_x_front: x_f,
sann.input_x_behind: x_b,
sann.input_y: y_onehot,
sann.dropout_keep_prob: args.dropout_rate,
sann.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, sann.global_step, train_summary_op, sann.loss],
feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(val_loader, writer=None):
"""Evaluates model on a validation set."""
batches_validation = dh.batch_iter(
list(create_input_data(val_loader)), args.batch_size, 1)
eval_counter, eval_loss = 0, 0.0
true_labels = []
predicted_scores = []
predicted_labels = []
for batch_validation in batches_validation:
x_f, x_b, y_onehot = zip(*batch_validation)
feed_dict = {
sann.input_x_front: x_f,
sann.input_x_behind: x_b,
sann.input_y: y_onehot,
sann.dropout_keep_prob: 1.0,
sann.is_training: False
}
step, summaries, predictions, cur_loss = sess.run([
sann.global_step, validation_summary_op,
sann.topKPreds, sann.loss
], feed_dict)
# Prepare for calculating metrics
for i in y_onehot:
true_labels.append(np.argmax(i))
for j in predictions[0]:
predicted_scores.append(j[0])
for k in predictions[1]:
predicted_labels.append(k[0])
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1
eval_acc = accuracy_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels))
eval_pre = precision_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
eval_rec = recall_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
eval_F1 = f1_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(y_true=np.array(true_labels),
y_score=np.array(predicted_scores),
average='micro')
return eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc
# Generate batches
batches_train = dh.batch_iter(list(create_input_data(train_data)),
args.batch_size, args.epochs)
num_batches_per_epoch = int(
(len(train_data['f_pad_seqs']) - 1) / args.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
train_step(batch_train)
current_step = tf.train.global_step(sess, sann.global_step)
if current_step % args.evaluate_steps == 0:
logger.info("\nEvaluation:")
eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc = \
validation_step(val_data, writer=validation_summary_writer)
logger.info(
"All Validation set: Loss {0:g} | Acc {1:g} | Precision {2:g} | "
"Recall {3:g} | F1 {4:g} | AUC {5:g}".format(
eval_loss, eval_acc, eval_pre, eval_rec, eval_F1,
eval_auc))
best_saver.handle(eval_acc, sess, current_step)
if current_step % args.checkpoint_steps == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info("Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info(
"Epoch {0} has finished!".format(current_epoch))
logger.info("All Done.")
def train_han():
"""Training HAN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load sentences, labels, and training parameters
logger.info("Loading data...")
logger.info("Data processing...")
train_data = dh.load_data_and_labels(args.train_file, args.num_classes, args.word2vec_file, data_aug_flag=False)
val_data = dh.load_data_and_labels(args.validation_file, args.num_classes, args.word2vec_file, data_aug_flag=False)
logger.info("Data padding...")
x_train, y_train = dh.pad_data(train_data, args.pad_seq_len)
x_val, y_val = dh.pad_data(val_data, args.pad_seq_len)
# Build vocabulary
VOCAB_SIZE, EMBEDDING_SIZE, pretrained_word2vec_matrix = dh.load_word2vec_matrix(args.word2vec_file)
# Build a graph and han object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
han = TextHAN(
sequence_length=args.pad_seq_len,
vocab_size=VOCAB_SIZE,
embedding_type=args.embedding_type,
embedding_size=EMBEDDING_SIZE,
lstm_hidden_size=args.lstm_dim,
fc_hidden_size=args.fc_dim,
num_classes=args.num_classes,
l2_reg_lambda=args.l2_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(learning_rate=args.learning_rate,
global_step=han.global_step, decay_steps=args.decay_steps,
decay_rate=args.decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(han.loss))
grads, _ = tf.clip_by_global_norm(grads, clip_norm=args.norm_ratio)
train_op = optimizer.apply_gradients(zip(grads, vars), global_step=han.global_step, name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
out_dir = dh.get_out_dir(OPTION, logger)
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", han.loss)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries", "validation")
validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=args.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True)
if OPTION == 'R':
# Load han model
logger.info("Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
if OPTION == 'T':
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = args.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer, config)
# Save the embedding visualization
saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(han.global_step)
def train_step(x_batch, y_batch):
"""A single training step"""
feed_dict = {
han.input_x: x_batch,
han.input_y: y_batch,
han.dropout_keep_prob: args.dropout_rate,
han.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, han.global_step, train_summary_op, han.loss], feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_val, y_val, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(zip(x_val, y_val)), args.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * args.topK
eval_rec_tk = [0.0] * args.topK
eval_F1_tk = [0.0] * args.topK
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(args.topK)]
for batch_validation in batches_validation:
x_batch_val, y_batch_val = zip(*batch_validation)
feed_dict = {
han.input_x: x_batch_val,
han.input_y: y_batch_val,
han.dropout_keep_prob: 1.0,
han.is_training: False
}
step, summaries, scores, cur_loss = sess.run(
[han.global_step, validation_summary_op, han.scores, han.loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch_val:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=args.threshold)
for k in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(k)
# Predict by topK
for top_num in range(args.topK):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=scores, top_num=top_num+1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1
eval_pre_ts = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_rec_ts = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_F1_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
for top_num in range(args.topK):
eval_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F1_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
# Calculate the average PR
eval_prc = average_precision_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
return eval_loss, eval_auc, eval_prc, eval_pre_ts, eval_rec_ts, eval_F1_ts, \
eval_pre_tk, eval_rec_tk, eval_F1_tk
# Generate batches
batches_train = dh.batch_iter(
list(zip(x_train, y_train)), args.batch_size, args.epochs)
num_batches_per_epoch = int((len(x_train) - 1) / args.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train = zip(*batch_train)
train_step(x_batch_train, y_batch_train)
current_step = tf.train.global_step(sess, han.global_step)
if current_step % args.evaluate_steps == 0:
logger.info("\nEvaluation:")
eval_loss, eval_auc, eval_prc, \
eval_pre_ts, eval_rec_ts, eval_F1_ts, eval_pre_tk, eval_rec_tk, eval_F1_tk = \
validation_step(x_val, y_val, writer=validation_summary_writer)
logger.info("All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}"
.format(eval_loss, eval_auc, eval_prc))
# Predict by threshold
logger.info("Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}"
.format(eval_pre_ts, eval_rec_ts, eval_F1_ts))
# Predict by topK
logger.info("Predict by topK:")
for top_num in range(args.topK):
logger.info("Top{0}: Precision {1:g}, Recall {2:g}, F1 {3:g}"
.format(top_num+1, eval_pre_tk[top_num], eval_rec_tk[top_num], eval_F1_tk[top_num]))
best_saver.handle(eval_prc, sess, current_step)
if current_step % args.checkpoint_steps == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
logger.info("Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info("Epoch {0} has finished!".format(current_epoch))
logger.info("All Done.")
def train_tarnn():
"""Training TARNN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load sentences, labels, and training parameters
logger.info("Loading data...")
logger.info("Data processing...")
train_data = dh.load_data_and_labels(args.train_file, args.word2vec_file, data_aug_flag=False)
val_data = dh.load_data_and_labels(args.validation_file, args.word2vec_file, data_aug_flag=False)
logger.info("Data padding...")
x_train_content, x_train_question, x_train_option, y_train = dh.pad_data(train_data, args.pad_seq_len)
x_val_content, x_val_question, x_val_option, y_val = dh.pad_data(val_data, args.pad_seq_len)
# Build vocabulary
VOCAB_SIZE, EMBEDDING_SIZE, pretrained_word2vec_matrix = dh.load_word2vec_matrix(args.word2vec_file)
# Build a graph and tarnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
tarnn = TextTARNN(
sequence_length=args.pad_seq_len,
vocab_size=VOCAB_SIZE,
embedding_type=args.embedding_type,
embedding_size=EMBEDDING_SIZE,
rnn_hidden_size=args.rnn_dim,
rnn_type=args.rnn_type,
rnn_layers=args.rnn_layers,
attention_type=args.attention_type,
fc_hidden_size=args.fc_dim,
l2_reg_lambda=args.l2_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(learning_rate=args.learning_rate,
global_step=tarnn.global_step, decay_steps=args.decay_steps,
decay_rate=args.decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(tarnn.loss))
grads, _ = tf.clip_by_global_norm(grads, clip_norm=args.norm_ratio)
train_op = optimizer.apply_gradients(zip(grads, vars), global_step=tarnn.global_step, name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
out_dir = dh.get_out_dir(OPTION, logger)
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", tarnn.loss)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries", "validation")
validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=args.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=False)
if OPTION == 'R':
# Load tarnn model
logger.info("Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
if OPTION == 'T':
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = args.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer, config)
# Save the embedding visualization
saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(tarnn.global_step)
def train_step(x_batch_content, x_batch_question, x_batch_option, y_batch):
"""A single training step"""
feed_dict = {
tarnn.input_x_content: x_batch_content,
tarnn.input_x_question: x_batch_question,
tarnn.input_x_option: x_batch_option,
tarnn.input_y: y_batch,
tarnn.dropout_keep_prob: args.dropout_rate,
tarnn.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, tarnn.global_step, train_summary_op, tarnn.loss], feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_val_content, x_val_question, x_val_option, y_val, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(zip(x_val_content, x_val_question, x_val_option, y_val)),
args.batch_size, 1)
eval_counter, eval_loss = 0, 0.0
true_labels = []
predicted_scores = []
for batch_validation in batches_validation:
x_batch_content, x_batch_question, x_batch_option, y_batch = zip(*batch_validation)
feed_dict = {
tarnn.input_x_content: x_batch_content,
tarnn.input_x_question: x_batch_question,
tarnn.input_x_option: x_batch_option,
tarnn.input_y: y_batch,
tarnn.dropout_keep_prob: 1.0,
tarnn.is_training: False
}
step, summaries, scores, cur_loss = sess.run(
[tarnn.global_step, validation_summary_op, tarnn.scores, tarnn.loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch:
true_labels.append(i)
for j in scores:
predicted_scores.append(j)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate PCC & DOA
pcc, doa = dh.evaluation(true_labels, predicted_scores)
# Calculate RMSE
rmse = mean_squared_error(true_labels, predicted_scores) ** 0.5
r2 = r2_score(true_labels, predicted_scores)
return eval_loss, pcc, doa, rmse, r2
# Generate batches
batches_train = dh.batch_iter(list(zip(x_train_content, x_train_question, x_train_option, y_train)),
args.batch_size, args.epochs)
num_batches_per_epoch = int((len(y_train) - 1) / args.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train_content, x_batch_train_question, x_batch_train_option, y_batch_train = zip(*batch_train)
train_step(x_batch_train_content, x_batch_train_question, x_batch_train_option, y_batch_train)
current_step = tf.train.global_step(sess, tarnn.global_step)
if current_step % args.evaluate_steps == 0:
logger.info("\nEvaluation:")
eval_loss, pcc, doa, rmse, r2 = validation_step(x_val_content, x_val_question, x_val_option, y_val,
writer=validation_summary_writer)
logger.info("All Validation set: Loss {0:g} | PCC {1:g} | DOA {2:g} | RMSE {3:g} | R2 {4:g}"
.format(eval_loss, pcc, doa, rmse, r2))
best_saver.handle(rmse, sess, current_step)
if current_step % args.checkpoint_steps == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
logger.info("Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info("Epoch {0} has finished!".format(current_epoch))
logger.info("All Done.")
