def train_rnn():
"""Training RNN model."""
# Load sentences, labels, and training parameters
logger.info("✔︎ Loading data...")
logger.info("✔︎ Training data processing...")
train_data = dh.load_data_and_labels(FLAGS.training_data_file,
FLAGS.num_classes,
FLAGS.embedding_dim,
data_aug_flag=False)
logger.info("✔︎ Validation data processing...")
val_data = dh.load_data_and_labels(FLAGS.validation_data_file,
FLAGS.num_classes,
FLAGS.embedding_dim,
data_aug_flag=False)
logger.info("Recommended padding Sequence length is: {0}".format(
FLAGS.pad_seq_len))
logger.info("✔︎ Training data padding...")
x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info("✔︎ Validation data padding...")
x_val, y_val = dh.pad_data(val_data, FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE, pretrained_word2vec_matrix = dh.load_word2vec_matrix(
FLAGS.embedding_dim)
# Build a graph and rnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
rnn = TextRNN(sequence_length=FLAGS.pad_seq_len,
num_classes=FLAGS.num_classes,
vocab_size=VOCAB_SIZE,
lstm_hidden_size=FLAGS.lstm_hidden_size,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(
learning_rate=FLAGS.learning_rate,
global_step=rnn.global_step,
decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(rnn.loss))
grads, _ = tf.clip_by_global_norm(grads,
clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(
zip(grads, vars),
global_step=rnn.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input(
"☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): "
) # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input(
"✘ The format of your input is illegal, please re-input: "
)
logger.info(
"✔︎ The format of your input is legal, now loading to next step..."
)
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", rnn.loss)
# Train summaries
train_summary_op = tf.summary.merge(
[loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir,
num_to_keep=3,
maximize=True)
if FLAGS.train_or_restore == 'R':
# Load rnn model
logger.info("✔︎ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = FLAGS.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer,
config)
# Save the embedding visualization
saver.save(
sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(rnn.global_step)
def train_step(x_batch, y_batch):
"""A single training step"""
feed_dict = {
rnn.input_x: x_batch,
rnn.input_y: y_batch,
rnn.dropout_keep_prob: FLAGS.dropout_keep_prob,
rnn.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, rnn.global_step, train_summary_op, rnn.loss],
feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_val, y_val, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(zip(x_val, y_val)),
FLAGS.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * FLAGS.top_num
eval_rec_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_validation in batches_validation:
x_batch_val, y_batch_val = zip(*batch_validation)
feed_dict = {
rnn.input_x: x_batch_val,
rnn.input_y: y_batch_val,
rnn.dropout_keep_prob: 1.0,
rnn.is_training: False
}
step, summaries, scores, cur_loss = sess.run([
rnn.global_step, validation_summary_op, rnn.scores,
rnn.loss
], feed_dict)
# Prepare for calculating metrics
for i in y_batch_val:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold)
for k in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(k)
# Predict by topK
for top_num in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(
scores=scores, top_num=top_num + 1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1 (threshold & topK)
eval_pre_ts = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_rec_ts = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_F_ts = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
for top_num in range(FLAGS.top_num):
eval_pre_tk[top_num] = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F_tk[top_num] = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
# Calculate the average PR
eval_prc = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \
eval_rec_tk, eval_pre_tk, eval_F_tk
# Generate batches
batches_train = dh.batch_iter(list(zip(x_train, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int(
(len(x_train) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train = zip(*batch_train)
train_step(x_batch_train, y_batch_train)
current_step = tf.train.global_step(sess, rnn.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
eval_loss, eval_auc, eval_prc, \
eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk = \
validation_step(x_val, y_val, writer=validation_summary_writer)
logger.info(
"All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}"
.format(eval_loss, eval_auc, eval_prc))
# Predict by threshold
logger.info(
"☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F {2:g}"
.format(eval_pre_ts, eval_rec_ts, eval_F_ts))
# Predict by topK
logger.info("☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info(
"Top{0}: Precision {1:g}, Recall {2:g}, F {3:g}".
format(top_num + 1, eval_pre_tk[top_num],
eval_rec_tk[top_num], eval_F_tk[top_num]))
best_saver.handle(eval_prc, sess, current_step)
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info(
"✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info(
"✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
print("# Loading training data")
training_data_raw = open(config['TRAINING_DATA_LOCATION'],'r',encoding='latin-1').readlines()
random.shuffle(training_data_raw)
num_examples = config['NUM_EXAMPLES']
training_data_raw= training_data_raw[:num_examples]
print("# Processing training data")
x_train, y_train, vocab_processor = util.load_training_data(training_data_raw)
print(" Loading and Processing testing data")
testing_data_raw = open(config['TESTING_DATA_LOCATION'],'r',encoding='latin-1').readlines()
x_test, y_test = util.load_testing_data(testing_data_raw, vocab_processor)
print("# Creating RNN")
rnn = TextRNN(x_train.shape[1], y_train.shape[1], config['HIDDEN_LAYER_SIZE'],
len(vocab_processor.vocabulary_), config['WORD_VECTOR_DIM'], l2_reg=0.0)
optimizer = tf.train.AdamOptimizer(config['LEARNING_RATE'])
minimizer = optimizer.minimize(rnn.loss)
print("# Initializing Tensorflow")
init_op = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init_op)
saver = tf.train.Saver()
print("# Training")
batch_size = config['BATCH_SIZE']
no_of_batches = int(len(training_data_raw)/batch_size)
epoch = config['NUM_EPOCHS']
losses = []
def train_rnn():
"""Training RNN model."""
# Load sentences, labels, and training parameters
logger.info("✔︎ Loading data...")
logger.info("✔︎ Training data processing...")
train_data = dh.load_data_and_labels(FLAGS.training_data_file,
FLAGS.embedding_dim)
logger.info("✔︎ Validation data processing...")
validation_data = dh.load_data_and_labels(FLAGS.validation_data_file,
FLAGS.embedding_dim)
logger.info("Recommended padding Sequence length is: {0}".format(
FLAGS.pad_seq_len))
logger.info("✔︎ Training data padding...")
x_train_front, x_train_behind, y_train = dh.pad_data(
train_data, FLAGS.pad_seq_len)
logger.info("✔︎ Validation data padding...")
x_validation_front, x_validation_behind, y_validation = dh.pad_data(
validation_data, FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE, pretrained_word2vec_matrix = dh.load_word2vec_matrix(
FLAGS.embedding_dim)
# Build a graph and rnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
rnn = TextRNN(sequence_length=FLAGS.pad_seq_len,
num_classes=y_train.shape[1],
vocab_size=VOCAB_SIZE,
lstm_hidden_size=FLAGS.lstm_hidden_size,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(
learning_rate=FLAGS.learning_rate,
global_step=rnn.global_step,
decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(rnn.loss))
grads, _ = tf.clip_by_global_norm(grads,
clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(
zip(grads, vars),
global_step=rnn.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input(
"☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): "
) # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input(
"✘ The format of your input is illegal, please re-input: "
)
logger.info(
"✔︎ The format of your input is legal, now loading to next step..."
)
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", rnn.loss)
acc_summary = tf.summary.scalar("accuracy", rnn.accuracy)
# Train summaries
train_summary_op = tf.summary.merge(
[loss_summary, acc_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge(
[loss_summary, acc_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir,
num_to_keep=3,
maximize=True)
if FLAGS.train_or_restore == 'R':
# Load rnn model
logger.info("✔︎ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = FLAGS.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer,
config)
# Save the embedding visualization
saver.save(
sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(rnn.global_step)
def train_step(x_batch_front, x_batch_behind, y_batch):
"""A single training step"""
feed_dict = {
rnn.input_x_front: x_batch_front,
rnn.input_x_behind: x_batch_behind,
rnn.input_y: y_batch,
rnn.dropout_keep_prob: FLAGS.dropout_keep_prob,
rnn.is_training: True
}
_, step, summaries, loss, accuracy = sess.run([
train_op, rnn.global_step, train_summary_op, rnn.loss,
rnn.accuracy
], feed_dict)
logger.info("step {0}: loss {1:g}, acc {2:g}".format(
step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_batch_front,
x_batch_behind,
y_batch,
writer=None):
"""Evaluates model on a validation set"""
feed_dict = {
rnn.input_x_front: x_batch_front,
rnn.input_x_behind: x_batch_behind,
rnn.input_y: y_batch,
rnn.dropout_keep_prob: 1.0,
rnn.is_training: False
}
step, summaries, loss, accuracy, recall, precision, f1, auc = sess.run(
[
rnn.global_step, validation_summary_op, rnn.loss,
rnn.accuracy, rnn.recall, rnn.precision, rnn.F1,
rnn.AUC
], feed_dict)
logger.info(
"step {0}: loss {1:g}, acc {2:g}, recall {3:g}, precision {4:g}, f1 {5:g}, AUC {6}"
.format(step, loss, accuracy, recall, precision, f1, auc))
if writer:
writer.add_summary(summaries, step)
return accuracy
# Generate batches
batches = dh.batch_iter(
list(zip(x_train_front, x_train_behind, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int(
(len(x_train_front) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch in batches:
x_batch_front, x_batch_behind, y_batch = zip(*batch)
train_step(x_batch_front, x_batch_behind, y_batch)
current_step = tf.train.global_step(sess, rnn.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
accuracy = validation_step(
x_validation_front,
x_validation_behind,
y_validation,
writer=validation_summary_writer)
best_saver.handle(accuracy, sess, current_step)
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info(
"✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info(
"✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
def train_rnn():
"""Training RNN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load word2vec model
word2idx, embedding_matrix = dh.load_word2vec_matrix(args.word2vec_file)
# Load sentences, labels, and training parameters
logger.info("Loading data...")
logger.info("Data processing...")
train_data = dh.load_data_and_labels(args, args.train_file, word2idx)
val_data = dh.load_data_and_labels(args, args.validation_file, word2idx)
# Build a graph and rnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
rnn = TextRNN(sequence_length=args.pad_seq_len,
vocab_size=len(word2idx),
embedding_type=args.embedding_type,
embedding_size=args.embedding_dim,
lstm_hidden_size=args.lstm_dim,
fc_hidden_size=args.fc_dim,
num_classes=args.num_classes,
l2_reg_lambda=args.l2_lambda,
pretrained_embedding=embedding_matrix)
# Define training procedure
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(
learning_rate=args.learning_rate,
global_step=rnn.global_step,
decay_steps=args.decay_steps,
decay_rate=args.decay_rate,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(rnn.loss))
grads, _ = tf.clip_by_global_norm(grads,
clip_norm=args.norm_ratio)
train_op = optimizer.apply_gradients(
zip(grads, vars),
global_step=rnn.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
out_dir = dh.get_out_dir(OPTION, logger)
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", rnn.loss)
# Train summaries
train_summary_op = tf.summary.merge(
[loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=args.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir,
num_to_keep=3,
maximize=True)
if OPTION == 'R':
# Load rnn model
logger.info("Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
if OPTION == 'T':
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = args.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer,
config)
# Save the embedding visualization
saver.save(
sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(rnn.global_step)
def train_step(batch_data):
"""A single training step."""
x_f, x_b, y_onehot = zip(*batch_data)
feed_dict = {
rnn.input_x_front: x_f,
rnn.input_x_behind: x_b,
rnn.input_y: y_onehot,
rnn.dropout_keep_prob: args.dropout_rate,
rnn.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, rnn.global_step, train_summary_op, rnn.loss],
feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(val_loader, writer=None):
"""Evaluates model on a validation set."""
batches_validation = dh.batch_iter(
list(create_input_data(val_loader)), args.batch_size, 1)
eval_counter, eval_loss = 0, 0.0
true_labels = []
predicted_scores = []
predicted_labels = []
for batch_validation in batches_validation:
x_f, x_b, y_onehot = zip(*batch_validation)
feed_dict = {
rnn.input_x_front: x_f,
rnn.input_x_behind: x_b,
rnn.input_y: y_onehot,
rnn.dropout_keep_prob: 1.0,
rnn.is_training: False
}
step, summaries, predictions, cur_loss = sess.run([
rnn.global_step, validation_summary_op, rnn.topKPreds,
rnn.loss
], feed_dict)
# Prepare for calculating metrics
for i in y_onehot:
true_labels.append(np.argmax(i))
for j in predictions[0]:
predicted_scores.append(j[0])
for k in predictions[1]:
predicted_labels.append(k[0])
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1
eval_acc = accuracy_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels))
eval_pre = precision_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
eval_rec = recall_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
eval_F1 = f1_score(y_true=np.array(true_labels),
y_pred=np.array(predicted_labels),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(y_true=np.array(true_labels),
y_score=np.array(predicted_scores),
average='micro')
return eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc
# Generate batches
batches_train = dh.batch_iter(list(create_input_data(train_data)),
args.batch_size, args.epochs)
num_batches_per_epoch = int(
(len(train_data['f_pad_seqs']) - 1) / args.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
train_step(batch_train)
current_step = tf.train.global_step(sess, rnn.global_step)
if current_step % args.evaluate_steps == 0:
logger.info("\nEvaluation:")
eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc = \
validation_step(val_data, writer=validation_summary_writer)
logger.info(
"All Validation set: Loss {0:g} | Acc {1:g} | Precision {2:g} | "
"Recall {3:g} | F1 {4:g} | AUC {5:g}".format(
eval_loss, eval_acc, eval_pre, eval_rec, eval_F1,
eval_auc))
best_saver.handle(eval_acc, sess, current_step)
if current_step % args.checkpoint_steps == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info("Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info(
"Epoch {0} has finished!".format(current_epoch))
logger.info("All Done.")
# Parameters
learning_rate = 0.01
training_iters = 1000
batch_size = 64
display_step = 10
keep = 0.5
# Network Parameters
n_input = 1
n_steps = 56 # Sentence length
n_hidden = 256 # hidden layer num of features
n_classes = 2
embedding_size = 128
# Load model
myModel = TextRNN(learning_rate, n_input, n_steps, n_hidden, n_classes,
embedding_size, vocab_size, True, final_embeddings)
# Initializing the variables
init = tf.initialize_all_variables()
# Launch the graph
with tf.Session() as sess:
#Writing Directory information
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "rnn_runs", timestamp))
print("Writing to {}\n".format(out_dir))
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
sess1 = tf.Session(config=session_conf)
sess2 = tf.Session(config=session_conf)
with sess1.as_default() and sess2.as_default():
tf.set_random_seed(seed_value + number_of_network)
rnn = TextRNN(
sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=embedding_dimension,
l2_reg_lambda=FLAGS.l2_reg_lambda,
weights=class_weights_for_cross_entropy,
rel_pos_embedding_size=FLAGS.rel_pos_embedding_size,
rel_pos_cardinality=relative_positions_cardinality,
lstm_units=FLAGS.lstm_units,
pos_tags_embedding_size=FLAGS.pos_tags_embedding_size,
pos_tags_cardinality=pos_tags_cardinality,
with_eye_tracking=cfg["features"]["gaze"],
et_features_size=et.shape[2],
et_number_of_bins=cfg["features"]["binned"],
et_embedding_dimension=FLAGS.et_embedding_dimension,
with_eeg=cfg["features"]["eeg"],
eeg_features_size=eeg.shape[2],
use_normalization_layer=True)
cnn = TextCNN(
sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=embedding_dimension,