def train_rcnn():
"""Training RCNN model."""
# Load sentences, labels, and training parameters
logger.info("✔︎ Loading data...")
logger.info("✔︎ Training data processing...")
train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.num_classes,
FLAGS.embedding_dim, data_aug_flag=False)
logger.info("✔︎ Validation data processing...")
val_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes,
FLAGS.embedding_dim, data_aug_flag=False)
logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len))
logger.info("✔︎ Training data padding...")
x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info("✔︎ Validation data padding...")
x_val, y_val = dh.pad_data(val_data, FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim)
# Build a graph and rcnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
rcnn = TextRCNN(
sequence_length=FLAGS.pad_seq_len,
num_classes=FLAGS.num_classes,
vocab_size=VOCAB_SIZE,
lstm_hidden_size=FLAGS.lstm_hidden_size,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(','))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate,
global_step=rcnn.global_step, decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(rcnn.loss))
grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(zip(grads, vars), global_step=rcnn.global_step, name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input("☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): ") # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input("✘ The format of your input is illegal, please re-input: ")
logger.info("✔︎ The format of your input is legal, now loading to next step...")
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", rcnn.loss)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries", "validation")
validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True)
if FLAGS.train_or_restore == 'R':
# Load rcnn model
logger.info("✔︎ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = FLAGS.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer, config)
# Save the embedding visualization
saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(rcnn.global_step)
def train_step(x_batch, y_batch):
"""A single training step"""
feed_dict = {
rcnn.input_x: x_batch,
rcnn.input_y: y_batch,
rcnn.dropout_keep_prob: FLAGS.dropout_keep_prob,
rcnn.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, rcnn.global_step, train_summary_op, rcnn.loss], feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_val, y_val, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(zip(x_val, y_val)), FLAGS.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * FLAGS.top_num
eval_rec_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_validation in batches_validation:
x_batch_val, y_batch_val = zip(*batch_validation)
feed_dict = {
rcnn.input_x: x_batch_val,
rcnn.input_y: y_batch_val,
rcnn.dropout_keep_prob: 1.0,
rcnn.is_training: False
}
step, summaries, scores, cur_loss = sess.run(
[rcnn.global_step, validation_summary_op, rcnn.scores, rcnn.loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch_val:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold)
for k in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(k)
# Predict by topK
for top_num in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=scores, top_num=top_num+1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1 (threshold & topK)
eval_pre_ts = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_rec_ts = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_F_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
for top_num in range(FLAGS.top_num):
eval_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
# Calculate the average PR
eval_prc = average_precision_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \
eval_rec_tk, eval_pre_tk, eval_F_tk
# Generate batches
batches_train = dh.batch_iter(
list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int((len(x_train) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train = zip(*batch_train)
train_step(x_batch_train, y_batch_train)
current_step = tf.train.global_step(sess, rcnn.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
eval_loss, eval_auc, eval_prc, \
eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk = \
validation_step(x_val, y_val, writer=validation_summary_writer)
logger.info("All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}"
.format(eval_loss, eval_auc, eval_prc))
# Predict by threshold
logger.info("☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F {2:g}"
.format(eval_pre_ts, eval_rec_ts, eval_F_ts))
# Predict by topK
logger.info("☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info("Top{0}: Precision {1:g}, Recall {2:g}, F {3:g}"
.format(top_num+1, eval_pre_tk[top_num], eval_rec_tk[top_num], eval_F_tk[top_num]))
best_saver.handle(eval_prc, sess, current_step)
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
logger.info("✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info("✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
def train_cnn():
"""Training CNN model."""
# Load sentences, labels, and training parameters
logger.info("✔︎ Loading data...")
logger.info("✔︎ Training data processing...")
train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.embedding_dim)
logger.info("✔︎ Validation data processing...")
validation_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.embedding_dim)
logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len))
logger.info("✔︎ Training data padding...")
x_train_front, x_train_behind, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info("✔︎ Validation data padding...")
x_validation_front, x_validation_behind, y_validation = dh.pad_data(validation_data, FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim)
# Build a graph and cnn object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(
sequence_length=FLAGS.pad_seq_len,
num_classes=y_train.shape[1],
vocab_size=VOCAB_SIZE,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(','))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate,
global_step=cnn.global_step, decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(cnn.loss))
grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(zip(grads, vars), global_step=cnn.global_step, name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input("☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): ") # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input("✘ The format of your input is illegal, please re-input: ")
logger.info("✔︎ The format of your input is legal, now loading to next step...")
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary, acc_summary])
validation_summary_dir = os.path.join(out_dir, "summaries", "validation")
validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True)
if FLAGS.train_or_restore == 'R':
# Load cnn model
logger.info("✔︎ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = "embedding"
embedding_conf.metadata_path = FLAGS.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer, config)
# Save the embedding visualization
saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(cnn.global_step)
def train_step(x_batch_front, x_batch_behind, y_batch):
"""A single training step"""
feed_dict = {
cnn.input_x_front: x_batch_front,
cnn.input_x_behind: x_batch_behind,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: FLAGS.dropout_keep_prob,
cnn.is_training: True
}
_, step, summaries, loss, accuracy = sess.run(
[train_op, cnn.global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict)
logger.info("step {0}: loss {1:g}, acc {2:g}".format(step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_batch_front, x_batch_behind, y_batch, writer=None):
"""Evaluates model on a validation set"""
feed_dict = {
cnn.input_x_front: x_batch_front,
cnn.input_x_behind: x_batch_behind,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0,
cnn.is_training: False
}
step, summaries, loss, accuracy, recall, precision, f1, auc = sess.run(
[cnn.global_step, validation_summary_op, cnn.loss, cnn.accuracy,
cnn.recall, cnn.precision, cnn.F1, cnn.AUC], feed_dict)
logger.info("step {0}: loss {1:g}, acc {2:g}, recall {3:g}, precision {4:g}, f1 {5:g}, AUC {6}"
.format(step, loss, accuracy, recall, precision, f1, auc))
if writer:
writer.add_summary(summaries, step)
return accuracy
# Generate batches
batches = dh.batch_iter(
list(zip(x_train_front, x_train_behind, y_train)), FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int((len(x_train_front) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch in batches:
x_batch_front, x_batch_behind, y_batch = zip(*batch)
train_step(x_batch_front, x_batch_behind, y_batch)
current_step = tf.train.global_step(sess, cnn.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
accuracy = validation_step(x_validation_front, x_validation_behind, y_validation,
writer=validation_summary_writer)
best_saver.handle(accuracy, sess, current_step)
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
logger.info("✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info("✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
def train_mann():
"""Training MANN model."""
# Load sentences, labels, and training parameters
logger.info('✔︎ Loading data...')
logger.info('✔︎ Training data processing...')
train_data = dh.load_data_and_labels(FLAGS.training_data_file,
FLAGS.num_classes,
FLAGS.embedding_dim)
logger.info('✔︎ Validation data processing...')
validation_data = \
dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes, FLAGS.embedding_dim)
logger.info('Recommended padding Sequence length is: {0}'.format(
FLAGS.pad_seq_len))
logger.info('✔︎ Training data padding...')
x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info('✔︎ Validation data padding...')
x_validation, y_validation = dh.pad_data(validation_data,
FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(
VOCAB_SIZE, FLAGS.embedding_dim)
# Build a graph and mann object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
mann = TextMANN(sequence_length=FLAGS.pad_seq_len,
num_classes=FLAGS.num_classes,
batch_size=FLAGS.batch_size,
vocab_size=VOCAB_SIZE,
lstm_hidden_size=FLAGS.lstm_hidden_size,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define training procedure
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(
learning_rate=FLAGS.learning_rate,
global_step=mann.global_step,
decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(mann.loss))
grads, _ = tf.clip_by_global_norm(grads,
clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(
zip(grads, vars),
global_step=mann.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input(
"☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): "
) # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input(
'✘ The format of your input is illegal, please re-input: '
)
logger.info(
'✔︎ The format of your input is legal, now loading to next step...'
)
checkpoint_dir = 'runs/' + MODEL + '/checkpoints/'
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", mann.loss)
# Train summaries
train_summary_op = tf.summary.merge(
[loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
if FLAGS.train_or_restore == 'R':
# Load mann model
logger.info("✔ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Embedding visualization config
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = 'embedding'
embedding_conf.metadata_path = FLAGS.metadata_file
projector.visualize_embeddings(train_summary_writer, config)
projector.visualize_embeddings(validation_summary_writer,
config)
# Save the embedding visualization
saver.save(
sess, os.path.join(out_dir, 'embedding', 'embedding.ckpt'))
current_step = sess.run(mann.global_step)
def train_step(x_batch, y_batch):
"""A single training step"""
feed_dict = {
mann.input_x: x_batch,
mann.input_y: y_batch,
mann.dropout_keep_prob: FLAGS.dropout_keep_prob,
mann.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, mann.global_step, train_summary_op, mann.loss],
feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_validation, y_validation, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(
list(zip(x_validation, y_validation)), FLAGS.batch_size,
FLAGS.num_epochs)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts = 0, 0.0, 0.0, 0.0, 0.0
eval_rec_tk = [0.0] * FLAGS.top_num
eval_acc_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
for batch_validation in batches_validation:
x_batch_validation, y_batch_validation = zip(
*batch_validation)
feed_dict = {
mann.input_x: x_batch_validation,
mann.input_y: y_batch_validation,
mann.dropout_keep_prob: 1.0,
mann.is_training: False
}
step, summaries, scores, cur_loss = sess.run([
mann.global_step, validation_summary_op, mann.scores,
mann.loss
], feed_dict)
# Predict by threshold
predicted_labels_threshold, predicted_values_threshold = \
dh.get_label_using_scores_by_threshold(scores=scores, threshold=FLAGS.threshold)
cur_rec_ts, cur_acc_ts, cur_F_ts = 0.0, 0.0, 0.0
for index, predicted_label_threshold in enumerate(
predicted_labels_threshold):
rec_inc_ts, acc_inc_ts, F_inc_ts = dh.cal_metric(
predicted_label_threshold,
y_batch_validation[index])
cur_rec_ts, cur_acc_ts, cur_F_ts = cur_rec_ts + rec_inc_ts, \
cur_acc_ts + acc_inc_ts, \
cur_F_ts + F_inc_ts
cur_rec_ts = cur_rec_ts / len(y_batch_validation)
cur_acc_ts = cur_acc_ts / len(y_batch_validation)
cur_F_ts = cur_F_ts / len(y_batch_validation)
eval_rec_ts, eval_acc_ts, eval_F_ts = eval_rec_ts + cur_rec_ts, \
eval_acc_ts + cur_acc_ts, \
eval_F_ts + cur_F_ts
# Predict by topK
topK_predicted_labels = []
for top_num in range(FLAGS.top_num):
predicted_labels_topk, predicted_values_topk = \
dh.get_label_using_scores_by_topk(scores=scores, top_num=top_num+1)
topK_predicted_labels.append(predicted_labels_topk)
cur_rec_tk = [0.0] * FLAGS.top_num
cur_acc_tk = [0.0] * FLAGS.top_num
cur_F_tk = [0.0] * FLAGS.top_num
for top_num, predicted_labels_topK in enumerate(
topK_predicted_labels):
for index, predicted_label_topK in enumerate(
predicted_labels_topK):
rec_inc_tk, acc_inc_tk, F_inc_tk = dh.cal_metric(
predicted_label_topK,
y_batch_validation[index])
cur_rec_tk[top_num], cur_acc_tk[top_num], cur_F_tk[top_num] = \
cur_rec_tk[top_num] + rec_inc_tk, \
cur_acc_tk[top_num] + acc_inc_tk, \
cur_F_tk[top_num] + F_inc_tk
cur_rec_tk[top_num] = cur_rec_tk[top_num] / len(
y_batch_validation)
cur_acc_tk[top_num] = cur_acc_tk[top_num] / len(
y_batch_validation)
cur_F_tk[top_num] = cur_F_tk[top_num] / len(
y_batch_validation)
eval_rec_tk[top_num], eval_acc_tk[top_num], eval_F_tk[top_num] = \
eval_rec_tk[top_num] + cur_rec_tk[top_num], \
eval_acc_tk[top_num] + cur_acc_tk[top_num], \
eval_F_tk[top_num] + cur_F_tk[top_num]
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
logger.info("✔︎ validation batch {0}: loss {1:g}".format(
eval_counter, cur_loss))
logger.info(
"︎☛ Predict by threshold: recall {0:g}, accuracy {1:g}, F {2:g}"
.format(cur_rec_ts, cur_acc_ts, cur_F_ts))
logger.info("︎☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info(
"Top{0}: recall {1:g}, accuracy {2:g}, F {3:g}".
format(top_num + 1, cur_rec_tk[top_num],
cur_acc_tk[top_num], cur_F_tk[top_num]))
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
eval_rec_ts = float(eval_rec_ts / eval_counter)
eval_acc_ts = float(eval_acc_ts / eval_counter)
eval_F_ts = float(eval_F_ts / eval_counter)
for top_num in range(FLAGS.top_num):
eval_rec_tk[top_num] = float(eval_rec_tk[top_num] /
eval_counter)
eval_acc_tk[top_num] = float(eval_acc_tk[top_num] /
eval_counter)
eval_F_tk[top_num] = float(eval_F_tk[top_num] /
eval_counter)
return eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts, eval_rec_tk, eval_acc_tk, eval_F_tk
# Generate batches
batches_train = dh.batch_iter(list(zip(x_train, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int(
(len(x_train) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train = zip(*batch_train)
train_step(x_batch_train, y_batch_train)
current_step = tf.train.global_step(sess, mann.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts, eval_rec_tk, eval_acc_tk, eval_F_tk = \
validation_step(x_validation, y_validation, writer=validation_summary_writer)
logger.info(
"All Validation set: Loss {0:g}".format(eval_loss))
# Predict by threshold
logger.info(
"︎☛ Predict by threshold: Recall {0:g}, Accuracy {1:g}, F {2:g}"
.format(eval_rec_ts, eval_acc_ts, eval_F_ts))
# Predict by topK
logger.info("︎☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info(
"Top{0}: Recall {1:g}, Accuracy {2:g}, F {3:g}".
format(top_num + 1, eval_rec_tk[top_num],
eval_acc_tk[top_num], eval_F_tk[top_num]))
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info(
"✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info(
"✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
def train_fasttext():
"""Training FASTTEXT model."""
# Load sentences, labels, and training parameters
logger.info('✔︎ Loading data...')
logger.info('✔︎ Training data processing...')
train_data = dh.load_data_and_labels(FLAGS.training_data_file,
FLAGS.num_classes,
FLAGS.embedding_dim)
logger.info('✔︎ Validation data processing...')
validation_data = \
dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes, FLAGS.embedding_dim)
logger.info('Recommended padding Sequence length is: {0}'.format(
FLAGS.pad_seq_len))
logger.info('✔︎ Training data padding...')
x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info('✔︎ Validation data padding...')
x_validation, y_validation = dh.pad_data(validation_data,
FLAGS.pad_seq_len)
y_validation_bind = validation_data.labels_bind
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(
VOCAB_SIZE, FLAGS.embedding_dim)
# Build a graph and fasttext object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
fasttext = TextFAST(
sequence_length=FLAGS.pad_seq_len,
num_classes=FLAGS.num_classes,
vocab_size=VOCAB_SIZE,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix)
# Define Training procedure
# learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate, global_step=cnn.global_step,
# decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate,
# staircase=True)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads_and_vars = optimizer.compute_gradients(fasttext.loss)
train_op = optimizer.apply_gradients(
grads_and_vars,
global_step=fasttext.global_step,
name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{0}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input(
"☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): "
) # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input(
'✘ The format of your input is illegal, please re-input: '
)
logger.info(
'✔︎ The format of your input is legal, now loading to next step...'
)
checkpoint_dir = 'runs/' + MODEL + '/checkpoints/'
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", fasttext.loss)
# acc_summary = tf.summary.scalar("accuracy", fasttext.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge(
[loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries",
"validation")
validation_summary_writer = tf.summary.FileWriter(
validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
if FLAGS.train_or_restore == 'R':
# Load fasttext model
logger.info("✔ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
current_step = sess.run(fasttext.global_step)
def train_step(x_batch, y_batch):
"""A single training step"""
feed_dict = {
fasttext.input_x: x_batch,
fasttext.input_y: y_batch,
fasttext.dropout_keep_prob: FLAGS.dropout_keep_prob,
fasttext.is_training: True
}
_, step, summaries, loss = sess.run([
train_op, fasttext.global_step, train_summary_op,
fasttext.loss
], feed_dict)
time_str = datetime.datetime.now().isoformat()
logger.info("{0}: step {1}, loss {2:g}".format(
time_str, step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_validation,
y_validation,
y_validation_bind,
writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(
list(zip(x_validation, y_validation, y_validation_bind)),
FLAGS.batch_size, FLAGS.num_epochs)
eval_loss, eval_rec, eval_acc, eval_counter = 0.0, 0.0, 0.0, 0
for batch_validation in batches_validation:
x_batch_validation, y_batch_validation, y_batch_validation_bind = zip(
*batch_validation)
feed_dict = {
fasttext.input_x: x_batch_validation,
fasttext.input_y: y_batch_validation,
fasttext.dropout_keep_prob: 1.0,
fasttext.is_training: False
}
step, summaries, logits, cur_loss = sess.run([
fasttext.global_step, validation_summary_op,
fasttext.logits, fasttext.loss
], feed_dict)
if FLAGS.use_classbind_or_not == 'Y':
predicted_labels = dh.get_label_using_logits_and_classbind(
logits,
y_batch_validation_bind,
top_number=FLAGS.top_num)
if FLAGS.use_classbind_or_not == 'N':
predicted_labels = dh.get_label_using_logits(
logits, top_number=FLAGS.top_num)
cur_rec, cur_acc = 0.0, 0.0
for index, predicted_label in enumerate(predicted_labels):
rec_inc, acc_inc = dh.cal_rec_and_acc(
predicted_label, y_batch_validation[index])
cur_rec, cur_acc = cur_rec + rec_inc, cur_acc + acc_inc
cur_rec = cur_rec / len(y_batch_validation)
cur_acc = cur_acc / len(y_batch_validation)
eval_loss, eval_rec, eval_acc, eval_counter = eval_loss + cur_loss, eval_rec + cur_rec, \
eval_acc + cur_acc, eval_counter + 1
logger.info("✔︎ validation batch {0} finished.".format(
eval_counter))
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
eval_rec = float(eval_rec / eval_counter)
eval_acc = float(eval_acc / eval_counter)
return eval_loss, eval_rec, eval_acc
# Generate batches
batches_train = dh.batch_iter(list(zip(x_train, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train = zip(*batch_train)
train_step(x_batch_train, y_batch_train)
current_step = tf.train.global_step(sess, fasttext.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
eval_loss, eval_rec, eval_acc = validation_step(
x_validation,
y_validation,
y_validation_bind,
writer=validation_summary_writer)
time_str = datetime.datetime.now().isoformat()
logger.info(
"{0}: step {1}, loss {2:g}, rec {3:g}, acc {4:g}".
format(time_str, current_step, eval_loss, eval_rec,
eval_acc))
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logger.info(
"✔︎ Saved model checkpoint to {0}\n".format(path))
logger.info("✔︎ Done.")
def test_cnn():
"""Test CNN model."""
# Load data
logger.info("✔ Loading data...")
logger.info('Recommended padding Sequence length is: {0}'.format(
FLAGS.pad_seq_len))
logger.info('✔︎ Test data processing...')
test_data = dh.load_data_and_labels(FLAGS.test_data_file,
FLAGS.num_classes, FLAGS.embedding_dim)
logger.info('✔︎ Test data padding...')
x_test, y_test = dh.pad_data(test_data, FLAGS.pad_seq_len)
y_test_bind = test_data.labels_bind
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(
VOCAB_SIZE, FLAGS.embedding_dim)
# Load cnn model
logger.info("✔ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
# input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
# pre-trained_word2vec
pretrained_embedding = graph.get_operation_by_name(
"embedding/embedding").outputs[0]
# Tensors we want to evaluate
logits = graph.get_operation_by_name("output/logits").outputs[0]
# Generate batches for one epoch
batches = dh.batch_iter(list(zip(x_test, y_test, y_test_bind)),
FLAGS.batch_size,
1,
shuffle=False)
# Collect the predictions here
all_predicitons = []
eval_loss, eval_rec, eval_acc, eval_counter = 0.0, 0.0, 0.0, 0
for batch_test in batches:
x_batch_test, y_batch_test, y_batch_test_bind = zip(
*batch_test)
feed_dict = {input_x: x_batch_test, dropout_keep_prob: 1.0}
batch_logits = sess.run(logits, feed_dict)
if FLAGS.use_classbind_or_not == 'Y':
predicted_labels = dh.get_label_using_logits_and_classbind(
batch_logits,
y_batch_test_bind,
top_number=FLAGS.top_num)
if FLAGS.use_classbind_or_not == 'N':
predicted_labels = dh.get_label_using_logits(
batch_logits, top_number=FLAGS.top_num)
all_predicitons = np.append(all_predicitons, predicted_labels)
cur_rec, cur_acc = 0.0, 0.0
for index, predicted_label in enumerate(predicted_labels):
rec_inc, acc_inc = dh.cal_rec_and_acc(
predicted_label, y_batch_test[index])
cur_rec, cur_acc = cur_rec + rec_inc, cur_acc + acc_inc
cur_rec = cur_rec / len(y_batch_test)
cur_acc = cur_acc / len(y_batch_test)
eval_rec, eval_acc, eval_counter = eval_rec + cur_rec, eval_acc + cur_acc, eval_counter + 1
logger.info(
"✔︎ validation batch {0} finished.".format(eval_counter))
eval_rec = float(eval_rec / eval_counter)
eval_acc = float(eval_acc / eval_counter)
logger.info("☛ Recall {0:g}, Accuracy {1:g}".format(
eval_rec, eval_acc))
np.savetxt(SAVE_FILE, list(zip(all_predicitons)), fmt='%s')
logger.info("✔ Done.")
def train_lmlp():
"""Training LMLP model."""
# Load sentences, labels, and training parameters
logger.info("✔︎ Loading data...")
logger.info("✔︎ Training data processing...")
train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.num_classes_list, FLAGS.total_classes,
FLAGS.embedding_dim, data_aug_flag=False)
logger.info("✔︎ Validation data processing...")
val_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes_list, FLAGS.total_classes,
FLAGS.embedding_dim, data_aug_flag=False)
logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len))
logger.info("✔︎ Training data padding...")
x_train, y_train, y_train_tuple = dh.pad_data(train_data, FLAGS.pad_seq_len)
logger.info("✔︎ Validation data padding...")
x_val, y_val, y_val_tuple = dh.pad_data(val_data, FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim)
# Build a graph and lmlp object
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
lmlp = eval(FLAGS.lmlp_type)(
sequence_length=FLAGS.pad_seq_len,
num_classes_list=list(map(int, FLAGS.num_classes_list.split(','))),
total_classes=FLAGS.total_classes,
vocab_size=VOCAB_SIZE,
fc_hidden_size=FLAGS.fc_hidden_size,
embedding_size=FLAGS.embedding_dim,
embedding_type=FLAGS.embedding_type,
l2_reg_lambda=FLAGS.l2_reg_lambda,
pretrained_embedding=pretrained_word2vec_matrix,
alpha=FLAGS.alpha,
beta=FLAGS.beta)
# Define training procedure
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate,
global_step=lmlp.global_step, decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, vars = zip(*optimizer.compute_gradients(lmlp.loss))
grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio)
train_op = optimizer.apply_gradients(zip(grads, vars), global_step=lmlp.global_step, name="train_op")
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in zip(grads, vars):
if g is not None:
grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
if FLAGS.train_or_restore == 'R':
MODEL = input("☛ Please input the checkpoints model you want to restore, "
"it should be like(1490175368): ") # The model you want to restore
while not (MODEL.isdigit() and len(MODEL) == 10):
MODEL = input("✘ The format of your input is illegal, please re-input: ")
logger.info("✔︎ The format of your input is legal, now loading to next step...")
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
else:
timestamp = str(int(time.time()))
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
logger.info("✔︎ Writing to {0}\n".format(out_dir))
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints"))
# Summaries for loss
loss_summary = tf.summary.scalar("loss", lmlp.loss)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Validation summaries
validation_summary_op = tf.summary.merge([loss_summary])
validation_summary_dir = os.path.join(out_dir, "summaries", "validation")
validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)
best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True)
if FLAGS.train_or_restore == 'R':
# Load lmlp model
logger.info("✔︎ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
logger.info(checkpoint_file)
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
else:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Save the embedding visualization
saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt"))
current_step = sess.run(lmlp.global_step)
def train_step(x_batch, y_batch, y_batch_tuple):
"""A single training step"""
y_batch_first = [i[0] for i in y_batch_tuple]
y_batch_second = [j[1] for j in y_batch_tuple]
y_batch_third = [k[2] for k in y_batch_tuple]
feed_dict = {
lmlp.input_x: x_batch,
lmlp.input_y_first: y_batch_first,
lmlp.input_y_second: y_batch_second,
lmlp.input_y_third: y_batch_third,
lmlp.input_y: y_batch,
lmlp.dropout_keep_prob: FLAGS.dropout_keep_prob,
lmlp.is_training: True
}
_, step, summaries, loss = sess.run(
[train_op, lmlp.global_step, train_summary_op, lmlp.loss], feed_dict)
logger.info("step {0}: loss {1:g}".format(step, loss))
train_summary_writer.add_summary(summaries, step)
def validation_step(x_val, y_val, y_val_tuple, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(
list(zip(x_val, y_val, y_val_tuple)), FLAGS.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss, eval_auc = 0, 0.0, 0.0
eval_rec_ts, eval_pre_ts, eval_F_ts = 0.0, 0.0, 0.0
eval_rec_tk = [0.0] * FLAGS.top_num
eval_pre_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
val_scores = []
for batch_validation in batches_validation:
x_batch_val, y_batch_val, y_batch_val_tuple = zip(*batch_validation)
y_batch_val_first = [i[0] for i in y_batch_val_tuple]
y_batch_val_second = [j[1] for j in y_batch_val_tuple]
y_batch_val_third = [k[2] for k in y_batch_val_tuple]
feed_dict = {
lmlp.input_x: x_batch_val,
lmlp.input_y_first: y_batch_val_first,
lmlp.input_y_second: y_batch_val_second,
lmlp.input_y_third: y_batch_val_third,
lmlp.input_y: y_batch_val,
lmlp.dropout_keep_prob: 1.0,
lmlp.is_training: False
}
step, summaries, scores, cur_loss = sess.run(
[lmlp.global_step, validation_summary_op, lmlp.scores, lmlp.loss], feed_dict)
for predicted_scores in scores:
val_scores.append(predicted_scores)
# Predict by threshold
predicted_labels_threshold, predicted_values_threshold = \
dh.get_label_using_scores_by_threshold(scores=scores, threshold=FLAGS.threshold)
cur_rec_ts, cur_pre_ts, cur_F_ts = 0.0, 0.0, 0.0
for index, predicted_label_threshold in enumerate(predicted_labels_threshold):
rec_inc_ts, pre_inc_ts = dh.cal_metric(predicted_label_threshold, y_batch_val[index])
cur_rec_ts, cur_pre_ts = cur_rec_ts + rec_inc_ts, cur_pre_ts + pre_inc_ts
cur_rec_ts = cur_rec_ts / len(y_batch_val)
cur_pre_ts = cur_pre_ts / len(y_batch_val)
cur_F_ts = dh.cal_F(cur_rec_ts, cur_pre_ts)
eval_rec_ts, eval_pre_ts = eval_rec_ts + cur_rec_ts, eval_pre_ts + cur_pre_ts
# Predict by topK
topK_predicted_labels = []
for top_num in range(FLAGS.top_num):
predicted_labels_topk, predicted_values_topk = \
dh.get_label_using_scores_by_topk(scores=scores, top_num=top_num+1)
topK_predicted_labels.append(predicted_labels_topk)
cur_rec_tk = [0.0] * FLAGS.top_num
cur_pre_tk = [0.0] * FLAGS.top_num
cur_F_tk = [0.0] * FLAGS.top_num
for top_num, predicted_labels_topK in enumerate(topK_predicted_labels):
for index, predicted_label_topK in enumerate(predicted_labels_topK):
rec_inc_tk, pre_inc_tk = dh.cal_metric(predicted_label_topK, y_batch_val[index])
cur_rec_tk[top_num], cur_pre_tk[top_num] = \
cur_rec_tk[top_num] + rec_inc_tk, cur_pre_tk[top_num] + pre_inc_tk
cur_rec_tk[top_num] = cur_rec_tk[top_num] / len(y_batch_val)
cur_pre_tk[top_num] = cur_pre_tk[top_num] / len(y_batch_val)
cur_F_tk[top_num] = dh.cal_F(cur_rec_tk[top_num], cur_pre_tk[top_num])
eval_rec_tk[top_num], eval_pre_tk[top_num] = \
eval_rec_tk[top_num] + cur_rec_tk[top_num], eval_pre_tk[top_num] + cur_pre_tk[top_num]
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
# Calculate the average AUC
val_scores = np.array(val_scores)
y_val = np.array(y_val)
missing_labels_num = 0
for index in range(FLAGS.total_classes):
y_true = y_val[:, index]
y_score = val_scores[:, index]
try:
eval_auc = eval_auc + roc_auc_score(y_true=y_true, y_score=y_score)
except:
missing_labels_num += 1
eval_auc = eval_auc / (FLAGS.total_classes - missing_labels_num)
eval_loss = float(eval_loss / eval_counter)
eval_rec_ts = float(eval_rec_ts / eval_counter)
eval_pre_ts = float(eval_pre_ts / eval_counter)
eval_F_ts = dh.cal_F(eval_rec_ts, eval_pre_ts)
for top_num in range(FLAGS.top_num):
eval_rec_tk[top_num] = float(eval_rec_tk[top_num] / eval_counter)
eval_pre_tk[top_num] = float(eval_pre_tk[top_num] / eval_counter)
eval_F_tk[top_num] = dh.cal_F(eval_rec_tk[top_num], eval_pre_tk[top_num])
return eval_loss, eval_auc, eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk
# Generate batches
batches_train = dh.batch_iter(
list(zip(x_train, y_train, y_train_tuple)), FLAGS.batch_size, FLAGS.num_epochs)
num_batches_per_epoch = int((len(x_train) - 1) / FLAGS.batch_size) + 1
# Training loop. For each batch...
for batch_train in batches_train:
x_batch_train, y_batch_train, y_batch_train_tuple = zip(*batch_train)
train_step(x_batch_train, y_batch_train, y_batch_train_tuple)
current_step = tf.train.global_step(sess, lmlp.global_step)
if current_step % FLAGS.evaluate_every == 0:
logger.info("\nEvaluation:")
eval_loss, eval_auc, eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk = \
validation_step(x_val, y_val, y_val_tuple, writer=validation_summary_writer)
logger.info("All Validation set: Loss {0:g} | AUC {1:g}".format(eval_loss, eval_auc))
# Predict by threshold
logger.info("☛ Predict by threshold: Recall {0:g}, Precision {1:g}, F {2:g}"
.format(eval_rec_ts, eval_pre_ts, eval_F_ts))
# Predict by topK
logger.info("☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info("Top{0}: Recall {1:g}, Precision {2:g}, F {3:g}"
.format(top_num+1, eval_rec_tk[top_num], eval_pre_tk[top_num], eval_F_tk[top_num]))
best_saver.handle(eval_auc, sess, current_step)
if current_step % FLAGS.checkpoint_every == 0:
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
logger.info("✔︎ Saved model checkpoint to {0}\n".format(path))
if current_step % num_batches_per_epoch == 0:
current_epoch = current_step // num_batches_per_epoch
logger.info("✔︎ Epoch {0} has finished!".format(current_epoch))
logger.info("✔︎ Done.")
def test_cnn():
"""Test CNN model."""
# Load data
logger.info("✔ Loading data...")
logger.info('Recommended padding Sequence length is: {0}'.format(
FLAGS.pad_seq_len))
logger.info('✔︎ Test data processing...')
test_data = dh.load_data_and_labels(FLAGS.test_data_file,
FLAGS.embedding_dim)
logger.info('✔︎ Test data padding...')
x_test_front, x_test_behind, y_test = dh.pad_data(test_data,
FLAGS.pad_seq_len)
# Build vocabulary
VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim)
pretrained_word2vec_matrix = dh.load_word2vec_matrix(
VOCAB_SIZE, FLAGS.embedding_dim)
# Load cnn model
logger.info("✔ Loading model...")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x_front = graph.get_operation_by_name(
"input_x_front").outputs[0]
input_x_behind = graph.get_operation_by_name(
"input_x_behind").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# pre-trained word2vec
pretrained_embedding = graph.get_operation_by_name(
"embedding/embedding").outputs[0]
# Tensors we want to evaluate
scores = graph.get_operation_by_name("output/scores").outputs
predictions = graph.get_operation_by_name(
"output/predictions").outputs[0]
softmax_scores = graph.get_operation_by_name(
"output/SoftMax_scores").outputs[0]
topKPreds = graph.get_operation_by_name(
"output/topKPreds").outputs[0]
accuracy = graph.get_operation_by_name(
"accuracy/accuracy").outputs[0]
loss = graph.get_operation_by_name("loss/loss").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = 'output/scores|output/predictions|output/SoftMax_scores|output/topKPreds'
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, output_node_names.split("|"))
tf.train.write_graph(output_graph_def,
'graph',
'graph-cnn-{0}.pb'.format(MODEL_LOG),
as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(
zip(x_test_front, x_test_behind, y_test)),
FLAGS.batch_size,
1,
shuffle=False)
# Collect the predictions here
all_scores = []
all_softmax_scores = []
all_predictions = []
all_topKPreds = []
for index, x_test_batch in enumerate(batches):
x_batch_front, x_batch_behind, y_batch = zip(*x_test_batch)
feed_dict = {
input_x_front: x_batch_front,
input_x_behind: x_batch_behind,
input_y: y_batch,
dropout_keep_prob: 1.0,
is_training: False
}
batch_scores = sess.run(scores, feed_dict)
all_scores = np.append(all_scores, batch_scores)
batch_softmax_scores = sess.run(softmax_scores, feed_dict)
all_softmax_scores = np.append(all_softmax_scores,
batch_softmax_scores)
batch_predictions = sess.run(predictions, feed_dict)
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
batch_topKPreds = sess.run(topKPreds, feed_dict)
all_topKPreds = np.append(all_topKPreds, batch_topKPreds)
batch_loss = sess.run(loss, feed_dict)
batch_acc = sess.run(accuracy, feed_dict)
logger.info(
"✔︎ Test batch {0}: loss {1:g}, accuracy {2:g}.".format(
(index + 1), batch_loss, batch_acc))
os.makedirs(SAVE_DIR)
np.savetxt(SAVE_DIR + '/result_sub_' + SUBSET + '.txt',
list(zip(all_predictions, all_topKPreds)),
fmt='%s')
logger.info("✔ Done.")