def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
prediction_dir = FLAGS.prediction_dir
if not tf.gfile.IsDirectory(prediction_dir):
tf.logging.info("Creating prediction directory: %s", prediction_dir)
tf.gfile.MakeDirs(prediction_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model = classifier_model.Classifier(model_config, mode="prediction")
model.build()
global MAX_NUM_TOKENS
MAX_NUM_TOKENS = model_config.sentence_length
init_op = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables())
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
start = time.time()
tf.logging.info("Starting prediction at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, init_op)
except KeyboardInterrupt:
pass
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
extraction_dir = FLAGS.extraction_dir
if not tf.gfile.IsDirectory(extraction_dir):
tf.logging.info("Creating extraction directory: %s", extraction_dir)
tf.gfile.MakeDirs(extraction_dir)
# generate eval dump file
dump_file = open(os.path.join(extraction_dir, FLAGS.extraction_file + '.json'), 'w')
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = classifier_model.Classifier(model_config, mode="extract")
model.build()
init_op = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables())
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
start = time.time()
tf.logging.info("Starting extraction at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, dump_file, init_op)
except KeyboardInterrupt:
dump_file.close()
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
eval_dir = FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s", eval_dir)
tf.gfile.MakeDirs(eval_dir)
# We are going to dump all evaluation metrics at each step into a file to have
# it easily accessible for later evaluation.
dump_fn = os.path.join(eval_dir, 'evaluation.json')
# Check if the dumpfile already exists
# If so, we initialize the global variables MAX_SCORE and GS_MAX_SCORE at first
if os.path.exists(dump_fn):
global MAX_SCORE, GS_MAX_SCORE
with open(dump_fn, 'r') as f:
for line in f:
curr = json.loads(line)
score = scoreIteration(curr)
if score >= MAX_SCORE:
MAX_SCORE = score
GS_MAX_SCORE = curr['global_step']
# generate eval dump file
dump_file = open(dump_fn, 'a')
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = classifier_model.Classifier(model_config, mode="eval")
model.build()
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
# Create the summary operation and the summary writer.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir)
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
while True:
start = time.time()
tf.logging.info(
"Starting evaluation at " +
time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op, dump_file)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time(
)
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
except KeyboardInterrupt:
dump_file.close()
def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model_config.autoencoder_checkpoint_dir = FLAGS.autoencoder_checkpoint_dir
training_config = configuration.TrainingConfig()
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = classifier_model.Classifier(
model_config, mode="train", train_encoder=FLAGS.train_encoder, use_pretrained_ae=True)
model.build()
# Set up the learning rate.
learning_rate_decay_fn = None
if FLAGS.finetune_model:
learning_rate = tf.constant(training_config.finetuning_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (training_config.num_examples_per_epoch /
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn,
variables=model.variables_to_train)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(
train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
save_interval_secs=60,
save_summaries_secs=60)
def main():
##############################################################
#
# FETCH TRAIN AND DEV EXAMPLES
#
##############################################################
# Get language_reference; the languages used in training ISR, which decides the original_label_onehots_tensor shape that Encoder will expect
language_reference_file = open(
os.path.join(FLAGS.isr_encoder_dir, "language_reference.json"), 'r')
language_reference = json.load(language_reference_file)
# Get train examples; either from raw_dataset or from bse_caches
train_language_abbreviations = util.parse_languages_into_abbreviation_list(
FLAGS.xnli_train_languages)
train_examples = util.get_xnli_train_examples(
FLAGS.data_dir, train_language_abbreviations)
random.shuffle(train_examples)
# Get dev examples; either from raw_dataset or from bse_caches
eval_language_abbreviations = util.parse_languages_into_abbreviation_list(
FLAGS.mid_train_xnli_eval_languages)
dev_examples_by_lang_dict = util.get_xnli_dev_examples_by_language(
FLAGS.data_dir, eval_language_abbreviations)
##############################################################
#
# LOAD ISR ENCODER
#
##############################################################
# Set up graph to extract ISR sentences
isr_encoder_graph = tf.Graph()
isr_sess = tf.Session(graph=isr_encoder_graph)
with isr_sess.as_default():
with isr_encoder_graph.as_default():
isr_model = "{}/{}".format(FLAGS.isr_encoder_dir,
FLAGS.isr_encoder_name)
imported_graph = tf.train.import_meta_graph(
"{}.meta".format(isr_model))
imported_graph.restore(isr_sess, isr_model)
# Placeholder tensors to pass in real data at each training step
original_sentences_tensor = isr_encoder_graph.get_tensor_by_name(
"original_sentences_tensor:0")
original_label_onehots_tensor = isr_encoder_graph.get_tensor_by_name(
"original_label_onehots_tensor:0")
### THE ISR TENSOR! ###
isr_sentences_tensor = isr_encoder_graph.get_tensor_by_name(
"forward_isr_sentences_tensor:0")
##############################################################
#
# BUILD XNLI CLASSIFIER
#
##############################################################
##### PREPROCESS INPUT SENTENCES #####
premise_x = tf.placeholder(tf.float32, [None, FLAGS.embedding_size])
hypothesis_x = tf.placeholder(tf.float32, [None, FLAGS.embedding_size])
y = tf.placeholder(tf.int32, [None])
keep_rate = 1.0 - FLAGS.dropout_rate
input_layer = tf.concat([premise_x, hypothesis_x], 1)
classifier = classifier_model.Classifier(
input_layer.shape[-1].value,
3) # 3 for one of either entailment/contradiction/neutral
logits, predictions_tensor = classifier(input_layer, keep_rate)
##### LOSS AND OPTIMIZER #####
total_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
logits=logits))
cls_vars = [
cls_var for cls_var in tf.trainable_variables()
if cls_var.name.startswith('cls')
]
cls_optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
beta1=FLAGS.beta1,
beta2=FLAGS.beta2).minimize(
total_loss, var_list=cls_vars)
### LOSS TENSORBOARD ###
tf.summary.scalar('LOSS 1: classifier_loss',
total_loss,
collections=['loss'])
### EVALUATION ACCURACY TENSORBOARD ###
# DISCRIMINIATOR CLASSIFIFER #
train_accuracy_tensor = tf.Variable(0.)
tf.summary.scalar("EVAL TRAIN: train_accuracy_tensor",
train_accuracy_tensor,
collections=['eval_accuracy'])
dev_accuracy_tensors_dict = {}
for eval_language_abbreviation in eval_language_abbreviations:
dev_accuracy_by_lang_tensor = tf.Variable(0.)
dev_accuracy_tensors_dict[
eval_language_abbreviation] = dev_accuracy_by_lang_tensor
tf.summary.scalar(
"EVAL DEV: {} eval_accuracy".format(eval_language_abbreviation),
dev_accuracy_by_lang_tensor,
collections=['eval_accuracy'])
##### SESSION CONFIGURATION #####
merged_loss_summary = tf.summary.merge_all(key='loss')
merged_eval_accuracy_summaries = tf.summary.merge_all(key='eval_accuracy')
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
os.system("mkdir -p {}".format(os.path.join(FLAGS.output_dir, "logs")))
train_writer = tf.summary.FileWriter(
os.path.join(FLAGS.output_dir, "logs"), sess.graph)
##############################################################
#
# TRAINING
#
##############################################################
# The remainder train examples from not cleanly divisible batch size will be omitted from training
global_step = 0
num_train_steps_per_epoch = int(
len(train_examples) / FLAGS.train_batch_size)
for epoch_num in range(1, FLAGS.num_train_epochs + 1):
for step_num in range(1, num_train_steps_per_epoch + 1):
######################################
##### Process Training Minibatch #####
######################################
minibatch_bse_premise_vectors, minibatch_bse_hypothesis_vectors, minibatch_labels, minibatch_languages = util.get_xnli_minibatch(
train_examples, step_num, FLAGS.train_batch_size,
language_reference)
minibatch_language_onehots = util.convert_to_onehots(
len(language_reference), minibatch_languages)
#############################
##### Get ISR Sentences #####
#############################
#### GET ISR SENTENCES #####
get_premise_isr_feed_dict = {
original_sentences_tensor: minibatch_bse_premise_vectors,
original_label_onehots_tensor: minibatch_language_onehots
}
get_hypothesis_isr_feed_dict = {
original_sentences_tensor: minibatch_bse_hypothesis_vectors,
original_label_onehots_tensor: minibatch_language_onehots
}
# forward pass through Generator's Encoder
minibatch_isr_premise_sentences = isr_sess.run(
isr_sentences_tensor, feed_dict=get_premise_isr_feed_dict)
minibatch_isr_hypothesis_sentences = isr_sess.run(
isr_sentences_tensor, feed_dict=get_hypothesis_isr_feed_dict)
##################################################
##### Perform Gradient Descent On Classifier #####
##################################################
##### RUN TRAINING WITH ISR SENTENCES #####
cls_feed_dict = {
premise_x: minibatch_isr_premise_sentences,
hypothesis_x: minibatch_isr_hypothesis_sentences,
y: minibatch_labels
}
_, current_loss, loss_summary, input_layer_val = sess.run(
[cls_optimizer, total_loss, merged_loss_summary, input_layer],
feed_dict=cls_feed_dict)
# Update loss information to Tensorboard
train_writer.add_summary(loss_summary, global_step=global_step)
############################
##### Save Checkpoints #####
############################
# Save checkpoint at every save_checkpoint_steps or at end of epoch
if (step_num % FLAGS.save_checkpoints_steps
== 0) or (step_num == num_train_steps_per_epoch):
saved_path = saver.save(
sess,
os.path.join(
FLAGS.output_dir,
"classifier-{}-{}".format(epoch_num, step_num)))
####################################
##### Run Mid Train Evaluation #####
####################################
if FLAGS.do_mid_train_eval:
# Run evaluation at every run_mid_train_eval_steps or at very beginning of training or at end of epoch
if (step_num % FLAGS.run_mid_train_eval_steps == 0) or (
(epoch_num == 1) and
(step_num == 1)) or (step_num
== num_train_steps_per_epoch):
# --------------------------------------------- #
###### Run Evaluation On Training Examples ######
# --------------------------------------------- #
train_examples_to_eval = None
if epoch_num == 1:
train_examples_to_eval = train_examples[:step_num *
FLAGS.
train_batch_size] # Only evaluate up to data trained on
else:
train_examples_to_eval = train_examples
train_accuracy = evaluate_model(
sess, isr_sess, original_sentences_tensor,
original_label_onehots_tensor, isr_sentences_tensor,
premise_x, hypothesis_x, predictions_tensor,
train_examples_to_eval, language_reference)
# Update Tensorboard summary tensor
sess.run(train_accuracy_tensor.assign(train_accuracy))
# ---------------------------------------------------------- #
###### Run Evaluation On Dev Examples For Each Language ######
# ---------------------------------------------------------- #
for eval_language_abbreviation in eval_language_abbreviations:
# Run evaluation on dev set
dev_examples_to_eval = dev_examples_by_lang_dict[
eval_language_abbreviation]
dev_accuracy_by_lang = evaluate_model(
sess, isr_sess, original_sentences_tensor,
original_label_onehots_tensor,
isr_sentences_tensor, premise_x, hypothesis_x,
predictions_tensor, dev_examples_to_eval,
language_reference)
# Update Tensorboard summary tensor for each eval language
sess.run(dev_accuracy_tensors_dict[
eval_language_abbreviation].assign(
dev_accuracy_by_lang))
# ------------------------------------------ #
###### Update Summary With Eval Results ######
# ------------------------------------------ #
eval_accuracy_summaries = sess.run(
merged_eval_accuracy_summaries)
train_writer.add_summary(eval_accuracy_summaries,
global_step=global_step)
# Increment global_step every training step
global_step += 1
##############################################################
#
# EXIT PROGRAM
#
##############################################################
# Close main tensorflow session and isr_sess
isr_sess.close()
sess.close()