def secondary_fn_tmp(hparams, identity, model_dir, model, eval_model, eval_sess,
name, worker_fn):
"""secondary helper function for inference and evaluation."""
steps_per_external_eval = 10
# initialize summary writer
summary_writer_path = os.path.join(hparams.out_dir, identity + name + "_log")
print("summary_writer_path", summary_writer_path)
summary_writer = tf.summary.FileWriter(summary_writer_path, model.graph)
config_proto = utils.get_config_proto(
log_device_placement=hparams.log_device_placement,
allow_soft_placement=True)
# create session
sess = tf.Session(config=config_proto, graph=model.graph)
# wait for the checkpoints
latest_ckpt = None
last_external_eval_step = 0
# main inference loop
while True:
latest_ckpt = tf.contrib.training.wait_for_new_checkpoint(
model_dir, latest_ckpt)
with model.graph.as_default():
_, global_step = model_helper.create_or_load_model(
model.model, model_dir, sess, name)
if global_step - last_external_eval_step >= steps_per_external_eval:
last_external_eval_step = global_step
worker_fn(model, sess, eval_model, eval_sess, latest_ckpt, summary_writer,
global_step, hparams)
if not hparams.eval_forever:
break # if eval_foever is disabled, we only evaluate once
summary_writer.close()
sess.close()
def eval(task, test_data):
data_seq, data_label0, data_label1 = gen_data(test_data, mode='train')
data_len = len(data_seq)
eval_step = int(math.ceil(float(data_len) / _hp.batch_size))
data_eval = [data_seq, data_label0, data_label1]
with tf.Graph().as_default() as graph:
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
model = Model(task, 'test')
model, global_step = _mh.create_or_load_model(
model, _hp.model_path, sess)
eval_tq = tqdm(range(eval_step))
l_total = 0.0
for eval_step_it in eval_tq:
batch = gen_train_data(data_eval, _hp.batch_size, eval_step_it)
if len(batch) == 0:
break
batch_data = data2ids(batch)
data_sess = batch_data[0]
subject_label = batch_data[1]
sentiment_label = batch_data[2]
label = model.model_exe(sess, "eval", data_sess)
label_result = None
if task == 'subject':
label_result = subject_label
else:
label_result = sentiment_label
acc_rate = eval_process(label_result, label)
eval_tq.set_description("%s acc: %f" % (task, acc_rate))
l_total = l_total + acc_rate
print("%s train set average acc: %f" %
(task, l_total / float(eval_step)))
def run_internal_eval(eval_model,
eval_sess,
model_dir,
hps,
summary_writer,
use_test_set=False):
"""Compute internal evaluation (perplexity) for both dev / test."""
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval")
dev_src_files, dev_tgt_files = data.get_files(hps.data_dir,
hps.test_prefix)
dev_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: dev_src_files,
eval_model.tgt_file_placeholder: dev_tgt_files
}
dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator, dev_eval_iterator_feed_dict,
summary_writer, "dev")
test_ppl = None
if use_test_set and hps.test_prefix:
test_src_files, test_tgt_files = data.get_files(
hps.data_dir, hps.test_prefix)
test_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: test_src_files,
eval_model.tgt_file_placeholder: test_tgt_files
}
test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator,
test_eval_iterator_feed_dict, summary_writer,
"test")
return dev_ppl, test_ppl
def run_external_eval(infer_model,
infer_sess,
model_dir,
hparams,
summary_writer,
save_best_dev=True,
use_test_set=True,
avg_ckpts=False):
"""
Compute external evaluation (bleu, rouge, etc.) for both dev / test.
"""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
dev_scores = None
test_scores = None
if global_step > 0:
utils.log("External evaluation, global step %d" % global_step)
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_infer_iterator_feed_dict = {
infer_model.src_placeholder: utils.load_data(dev_src_file),
infer_model.batch_size_placeholder: hparams.infer_batch_size,
}
dev_scores = external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
dev_infer_iterator_feed_dict,
dev_tgt_file,
"dev",
summary_writer,
save_on_best=save_best_dev,
avg_ckpts=avg_ckpts)
if use_test_set and hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_infer_iterator_feed_dict = {
infer_model.src_placeholder: utils.load_data(test_src_file),
infer_model.batch_size_placeholder: hparams.infer_batch_size,
}
test_scores = external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
test_infer_iterator_feed_dict,
test_tgt_file,
"test",
summary_writer,
save_on_best=False,
avg_ckpts=avg_ckpts)
return dev_scores, test_scores, global_step
def run_sample_decode(args, infer_model, infer_sess, model_dir, infer_data):
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
_sample_decode(args, loaded_infer_model, global_step, infer_sess,
infer_data)
def run_internal_eval(eval_model,
eval_sess,
model_dir,
hparams,
summary_writer,
use_test_set=True,
ckpt_index=None):
"""Compute internal evaluation (perplexity) for both dev / test."""
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval", ckpt_index)
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: dev_src_file,
eval_model.tgt_file_placeholder: dev_tgt_file
}
dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator, dev_eval_iterator_feed_dict,
summary_writer, "dev")
test_ppl = None
if use_test_set and hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: test_src_file,
eval_model.tgt_file_placeholder: test_tgt_file
}
test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator,
test_eval_iterator_feed_dict, summary_writer,
"test")
return dev_ppl, test_ppl
def run_internal_eval(eval_model,
eval_sess,
model_dir,
hparams,
summary_writer,
use_test_set=True,
dev_eval_iterator_feed_dict=None,
test_eval_iterator_feed_dict=None):
"""Compute internal evaluation (perplexity) for both dev / test.
Computes development and testing perplexities for given model.
Args:
eval_model: Evaluation model for which to compute perplexities.
eval_sess: Evaluation TensorFlow session.
model_dir: Directory from which to load evaluation model from.
hparams: Model hyper-parameters.
summary_writer: Summary writer for logging metrics to TensorBoard.
use_test_set: Computes testing perplexity if true; does not otherwise.
Note that the development perplexity is always computed regardless of
value of this parameter.
dev_eval_iterator_feed_dict: Feed dictionary for a TensorFlow session.
Can be used to pass in additional inputs necessary for running the
development evaluation.
test_eval_iterator_feed_dict: Feed dictionary for a TensorFlow session.
Can be used to pass in additional inputs necessary for running the
testing evaluation.
Returns:
Pair containing development perplexity and testing perplexity, in this
order.
"""
if dev_eval_iterator_feed_dict is None:
dev_eval_iterator_feed_dict = {}
if test_eval_iterator_feed_dict is None:
test_eval_iterator_feed_dict = {}
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval")
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_eval_iterator_feed_dict[eval_model.src_file_placeholder] = dev_src_file
dev_eval_iterator_feed_dict[eval_model.tgt_file_placeholder] = dev_tgt_file
dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator, dev_eval_iterator_feed_dict,
summary_writer, "dev")
test_ppl = None
if use_test_set and hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_eval_iterator_feed_dict[
eval_model.src_file_placeholder] = test_src_file
test_eval_iterator_feed_dict[
eval_model.tgt_file_placeholder] = test_tgt_file
test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator,
test_eval_iterator_feed_dict, summary_writer,
"test")
return dev_ppl, test_ppl
def start_sess_and_load_model(infer_model, ckpt_dir):
sess = tf.Session(
graph=infer_model.graph, config=misc.get_config_proto())
with infer_model.graph.as_default():
loaded_infer_model, global_step = helper.create_or_load_model(
infer_model.model, ckpt_dir, sess)
return sess, loaded_infer_model, global_step
def run_infer(infer_model, model_dir, infer_sess):
"""Load vars. then, run infer model for 1 episode."""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
model_dir, infer_model.model, infer_sess)
output_tuple = loaded_infer_model.infer(infer_sess)
return output_tuple
def run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, src_data, tgt_data):
"""Sample decode a random sentence from src_data."""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
_sample_decode(loaded_infer_model, global_step, infer_sess, hparams,
infer_model.iterator, src_data, tgt_data,
infer_model.src_placeholder, summary_writer)
def train_fn(hparams):
graph, model, iterator = create_train_model(hparams)
sess = tf.Session(graph=graph)
with graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
model, hparams.out_dir, sess, "train")
sess.run(model.iterator.initializer)
while global_step < hparams.num_train_steps:
start_time = time.time()
_, train_loss, train_accuracy, train_summary, global_step, _, _, _ = loaded_train_model.train(
sess)
print(train_loss, train_accuracy)
def run_external_eval(infer_model,
infer_sess,
model_dir,
hps,
summary_writer,
save_best_dev=True,
use_test_set=True):
"""Compute external evaluation (bleu, rouge, etc.) for both dev / test."""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
dev_src_file = hps.dev_src_file
dev_tgt_file = hps.dev_tgt_file
dev_infer_iterator_feed_dict = {
infer_model.src_placeholder:
inference_base_model.load_data(dev_src_file),
infer_model.batch_size_placeholder: hps.infer_batch_size,
}
dev_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hps,
infer_model.iterator,
dev_infer_iterator_feed_dict,
dev_tgt_file,
"dev",
summary_writer,
save_on_best=save_best_dev)
test_scores = None
if use_test_set and hps.test_prefix:
test_src_file = hps.test_src_file
test_tgt_file = hps.test_tgt_file
test_infer_iterator_feed_dict = {
infer_model.src_placeholder:
inference_base_model.load_data(test_src_file),
infer_model.batch_size_placeholder:
hps.infer_batch_size,
}
test_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hps,
infer_model.iterator,
test_infer_iterator_feed_dict,
test_tgt_file,
"test",
summary_writer,
save_on_best=False)
return dev_scores, test_scores, global_step
def run_external_eval(infer_model,
infer_sess,
model_dir,
hparams,
summary_writer,
save_best_dev=True):
"""Compute external evaluation (bleu, rouge, etc.) for both dev / test."""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_infer_iterator_feed_dict = {
infer_model.src_placeholder: inference.load_data(dev_src_file)
}
dev_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
dev_infer_iterator_feed_dict,
dev_tgt_file,
"dev",
summary_writer,
save_on_best=save_best_dev)
test_scores = None
if hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_infer_iterator_feed_dict = {
infer_model.src_placeholder: inference.load_data(test_src_file)
}
test_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
test_infer_iterator_feed_dict,
test_tgt_file,
"test",
summary_writer,
save_on_best=False)
return dev_scores, test_scores, global_step
def run_evaluation(eval_model, eval_sess, model_dir, input_eval_file,
output_eval_file, input_emb_weights, summary_writer):
with eval_model.graph.as_default():
# initialize the variables of the eval graph in eval_sess or load them from a checkpoint.
loaded_eval_model = model_helper.create_or_load_model(
eval_model.model, eval_sess, "eval", model_dir, input_emb_weights)
eval_iterator_feed_dict = {
eval_model.input_file_placeholder: input_eval_file,
eval_model.output_file_placeholder: output_eval_file
}
dev_loss = evaluation.eval(loaded_eval_model, eval_sess,
eval_model.iterator, eval_iterator_feed_dict)
model_helper.add_summary(summary_writer, "dev_loss", dev_loss)
return dev_loss
def run_internal_eval(eval_model, eval_handle, eval_sess, model_dir, hparams,
summary_writer):
"""Compute internal evaluation (perplexity) for dev."""
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval")
dev_eval_iterator_feed_dict = {
eval_model.data_file_placeholder: hparams.dev_data,
eval_model.kb_file_placeholder: hparams.dev_kb,
}
dev_ppl = _internal_eval(
loaded_eval_model, global_step, eval_sess, eval_model.eval_iterator,
dev_eval_iterator_feed_dict, eval_handle, summary_writer, "dev")
return dev_ppl, None
def _createTestInferCheckpoint(self, hparams, out_dir):
if not hparams.attention:
model_creator = nmt_model.Model
elif hparams.attention_architecture == "standard":
model_creator = attention_model.AttentionModel
elif hparams.attention_architecture in ["gnmt", "gnmt_v2"]:
model_creator = gnmt_model.GNMTModel
else:
raise ValueError("Unknown model architecture")
infer_model = model_helper.create_infer_model(model_creator, hparams)
with self.test_session(graph=infer_model.graph) as sess:
loaded_model, global_step = model_helper.create_or_load_model(
infer_model.model, out_dir, sess, "infer_name")
ckpt = loaded_model.saver.save(sess,
os.path.join(
out_dir, "translate.ckpt"),
global_step=global_step)
return ckpt
def infer(hparams, infer_model, infer_sess, batch_size=1):
src1_vocab_file = "%s/%s" % (hparams.data_dir, hparams.word_vocab)
src2_vocab_file = "%s/%s" % (hparams.data_dir, hparams.pos_vocab)
tgt_vocab_file = "%s/%s" % (hparams.data_dir, hparams.role_vocab)
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, hparams.ckpt_dir, infer_sess, "infer")
test_src1_file = "%s/%s" % (hparams.data_dir, hparams.test_word_data)
test_src2_file = "%s/%s" % (hparams.data_dir, hparams.test_pos_data)
test_src1_data = load_data(test_src1_file)
test_src2_data = load_data(test_src2_file)
test_tgt_file = "%s/%s_%d" % (hparams.output_dir,
hparams.test_role_data, global_step)
test_infer_iterator_feed_dict = {
infer_model.src1_placeholder: test_src1_data,
infer_model.src2_placeholder: test_src2_data,
infer_model.batch_size_placeholder: batch_size
}
print(" start infer..")
infer_sess.run(infer_model.iterator.initializer,
feed_dict=test_infer_iterator_feed_dict)
outfile = open(test_tgt_file, "w", encoding="utf-8")
infer_num = 200
while True and infer_num > 0:
try:
predict = loaded_infer_model.infer(infer_sess)
from_vocab, rev_from_vocab = data_utils.initialize_vocabulary(
src1_vocab_file)
_, rev_to_vocab = data_utils.initialize_vocabulary(
tgt_vocab_file)
sample_output = predict[0].tolist()
outputs = []
# alignment = alignment[0].tolist()
for output in sample_output:
outputs.append(tf.compat.as_str(rev_to_vocab[output]))
#print(" ".join(outputs))
outfile.write(" ".join(outputs) + "\n")
infer_num -= 1
except tf.errors.OutOfRangeError:
break
print(" infer done.")
outfile.close()
def _createTestInferCheckpoint(self, hparams, name):
# Prepare
hparams.vocab_prefix = ("nmt/testdata/test_infer_vocab")
hparams.src_vocab_file = hparams.vocab_prefix + "." + hparams.src
hparams.tgt_vocab_file = hparams.vocab_prefix + "." + hparams.tgt
out_dir = os.path.join(tf.test.get_temp_dir(), name)
os.makedirs(out_dir)
hparams.out_dir = out_dir
# Create check point
model_creator = inference.get_model_creator(hparams)
infer_model = model_helper.create_infer_model(model_creator, hparams)
with self.test_session(graph=infer_model.graph) as sess:
loaded_model, global_step = model_helper.create_or_load_model(
infer_model.model, out_dir, sess, "infer_name")
ckpt_path = loaded_model.saver.save(sess,
os.path.join(
out_dir, "translate.ckpt"),
global_step=global_step)
return ckpt_path
def run_internal_eval(args, eval_model, eval_sess, model_dir, eval_data):
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval")
eval_data_size = len(eval_data)
eval_data = utils.get_batches(eval_data, args.max_batch)
eval_total_loss = 0.0
total_predict_count = 0.0
for idx, batch in enumerate(eval_data):
eval_loss, predict_count, batch_size = loaded_eval_model.eval(
eval_sess, batch.user, batch.product, batch.rating,
batch.review_input, batch.review_output, batch.review_length)
eval_total_loss += eval_loss * batch_size
total_predict_count += predict_count
eval_avg_loss = eval_total_loss / eval_data_size
eval_ppl = np.exp(eval_total_loss / total_predict_count)
return eval_avg_loss, eval_ppl
def run_internal_eval(eval_model,
eval_sess,
model_dir,
hparams,
summary_writer,
use_test_set=True):
"""Compute internal evaluation (perplexity) for both dev / test."""
with eval_model.graph.as_default():
loaded_eval_model, global_step = model_helper.create_or_load_model(
eval_model.model, model_dir, eval_sess, "eval")
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: dev_src_file,
eval_model.tgt_file_placeholder: dev_tgt_file
}
dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator, dev_eval_iterator_feed_dict,
summary_writer, "dev")
test_ppl = None
if use_test_set and hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_eval_iterator_feed_dict = {
eval_model.src_file_placeholder: test_src_file,
eval_model.tgt_file_placeholder: test_tgt_file
}
test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess,
eval_model.iterator,
test_eval_iterator_feed_dict, summary_writer,
"test")
with open('forward_ppl.txt', 'a+') as fw:
fw.write('dev_ppl: ')
fw.write(str(round(dev_ppl, 2)))
fw.write('\ttest_ppl: ')
fw.write(str(round(test_ppl, 2)) + '\n')
return dev_ppl, test_ppl
def inference(task):
datas = get_data_list(mode='test')
test_data = gen_data(datas, mode='test')
test_len = len(test_data)
test_step = int(math.ceil(float(test_len) / _hp.batch_size))
with tf.Graph().as_default() as graph:
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
model = Model(task, 'test')
model, global_step = _mh.create_or_load_model(
model, _hp.model_path, sess)
label_out = []
for step in tqdm(range(test_step)):
data_test = gen_inference_data(test_data, _hp.batch_size, step)
data_test_ids = data2ids(data_test, mode='test')
label = model.model_exe(sess, "test", data_test_ids)
label = [label.values, label.indices] # TopKV2 to list
l = id2label(label, task, type='test')
label_out = label_out + l
csv_write(datas, label_out, task)
tf.train.write_graph(sess.graph_def, _hp.model_path,
'sentiment.pbtxt')
def run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, src_data, tgt_data):
"""
Sample decode a random sentence from src_data.
"""
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
# Pick a sentence and decode."""
decode_id = random.randint(0, len(src_data) - 1)
iterator_feed_dict = {
infer_model.src_placeholder: [src_data[decode_id]],
infer_model.batch_size_placeholder: 1,
}
infer_sess.run(infer_model.iterator.initializer,
feed_dict=iterator_feed_dict)
nmt_outputs, attention_summary = loaded_infer_model.decode(infer_sess)
if hparams.beam_width > 0:
# get the top translation.
nmt_outputs = nmt_outputs[0]
translation = utils.get_translation(nmt_outputs,
sent_id=0,
tgt_eos=hparams.eos)
utils.log("Sample src: {}".format(src_data[decode_id]))
utils.log("Sample ref: {}".format(tgt_data[decode_id]))
utils.log("NMT output: {}".format(translation))
# Summary
if attention_summary is not None:
summary_writer.add_summary(attention_summary, global_step)
def train(hparams, scope=None, target_session="", compute_ppl=0):
"""Train a translation model."""
log_device_placement = hparams.log_device_placement
out_dir = hparams.out_dir
num_train_steps = hparams.num_train_steps
steps_per_stats = hparams.steps_per_stats
steps_per_external_eval = hparams.steps_per_external_eval
steps_per_eval = 10 * steps_per_stats
avg_ckpts = hparams.avg_ckpts
if not steps_per_external_eval:
steps_per_external_eval = 5 * steps_per_eval
if not hparams.attention: # choose this model
model_creator = nmt_model.Model
else: # Attention
if (hparams.encoder_type == "gnmt"
or hparams.attention_architecture in ["gnmt", "gnmt_v2"]):
model_creator = gnmt_model.GNMTModel
elif hparams.attention_architecture == "standard":
model_creator = attention_model.AttentionModel
else:
raise ValueError("Unknown attention architecture %s" %
hparams.attention_architecture)
train_model = model_helper.create_train_model(model_creator, hparams,
scope)
eval_model = model_helper.create_eval_model(model_creator, hparams, scope)
infer_model = model_helper.create_infer_model(model_creator, hparams,
scope)
# Preload data for sample decoding.
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
sample_src_data = inference.load_data(dev_src_file)
sample_tgt_data = inference.load_data(dev_tgt_file)
wsd_src_file = "%s" % (hparams.sample_prefix)
wsd_src_data = inference.load_data(wsd_src_file)
summary_name = "train_log"
model_dir = hparams.out_dir
# Log and output files
log_file = os.path.join(out_dir, "log_%d" % time.time())
log_f = tf.gfile.GFile(log_file, mode="a")
utils.print_out("# log_file=%s" % log_file, log_f)
# TensorFlow model
config_proto = utils.get_config_proto(
log_device_placement=log_device_placement,
num_intra_threads=hparams.num_intra_threads,
num_inter_threads=hparams.num_inter_threads)
train_sess = tf.Session(target=target_session,
config=config_proto,
graph=train_model.graph)
eval_sess = tf.Session(target=target_session,
config=config_proto,
graph=eval_model.graph)
infer_sess = tf.Session(target=target_session,
config=config_proto,
graph=infer_model.graph)
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
# Summary writer
summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name),
train_model.graph)
# First evaluation
'''
run_full_eval(
model_dir, infer_model, infer_sess,
eval_model, eval_sess, hparams,
summary_writer, sample_src_data,
sample_tgt_data, avg_ckpts)
'''
last_stats_step = global_step
last_eval_step = global_step
last_external_eval_step = global_step
# This is the training loop.
stats, info, start_train_time = before_train(loaded_train_model,
train_model, train_sess,
global_step, hparams, log_f)
end_step = global_step + 100
while global_step < end_step: # num_train_steps
### Run a step ###
start_time = time.time()
try:
# then forward inference result to WSD, get reward
step_result = loaded_train_model.train(train_sess)
# forward reward to placeholder of loaded_train_model, and write a new train function where loss = loss*reward
hparams.epoch_step += 1
except tf.errors.OutOfRangeError:
# Finished going through the training dataset. Go to next epoch.
hparams.epoch_step = 0
utils.print_out(
"# Finished an epoch, step %d. Perform external evaluation" %
global_step)
# run_sample_decode(infer_model, infer_sess, model_dir, hparams,
# summary_writer, sample_src_data, sample_tgt_data)
# only for pretrain
# run_external_eval(infer_model, infer_sess, model_dir, hparams,
# summary_writer)
if avg_ckpts:
run_avg_external_eval(infer_model, infer_sess, model_dir,
hparams, summary_writer, global_step)
train_sess.run(train_model.iterator.initializer,
feed_dict={train_model.skip_count_placeholder: 0})
continue
# Process step_result, accumulate stats, and write summary
global_step, info["learning_rate"], step_summary = update_stats(
stats, start_time, step_result, hparams)
summary_writer.add_summary(step_summary, global_step)
if compute_ppl:
run_internal_eval(eval_model, eval_sess, model_dir, hparams,
summary_writer)
# Once in a while, we print statistics.
if global_step - last_stats_step >= steps_per_stats:
last_stats_step = global_step
is_overflow = process_stats(stats, info, global_step,
steps_per_stats, log_f)
print_step_info(" ", global_step, info,
_get_best_results(hparams), log_f)
if is_overflow:
break
# Reset statistics
stats = init_stats()
if global_step - last_eval_step >= steps_per_eval:
last_eval_step = global_step
utils.print_out("# Save eval, global step %d" % global_step)
utils.add_summary(summary_writer, global_step, "train_ppl",
info["train_ppl"])
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"translate.ckpt"),
global_step=global_step)
# Evaluate on dev/test
run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, sample_src_data, sample_tgt_data)
run_internal_eval(eval_model, eval_sess, model_dir, hparams,
summary_writer)
if global_step - last_external_eval_step >= steps_per_external_eval:
last_external_eval_step = global_step
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"translate.ckpt"),
global_step=global_step)
run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, sample_src_data, sample_tgt_data)
run_external_eval(infer_model, infer_sess, model_dir, hparams,
summary_writer)
if avg_ckpts:
run_avg_external_eval(infer_model, infer_sess, model_dir,
hparams, summary_writer, global_step)
# Done training
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir, "translate.ckpt"),
global_step=global_step)
'''
def train(hparams, scope=None, target_session=""):
"""Train a translation model."""
log_device_placement = hparams.log_device_placement
out_dir = hparams.out_dir
num_train_steps = hparams.num_train_steps
steps_per_stats = hparams.steps_per_stats
steps_per_external_eval = hparams.steps_per_external_eval
steps_per_eval = 10 * steps_per_stats
avg_ckpts = hparams.avg_ckpts
if not steps_per_external_eval:
steps_per_external_eval = 5 * steps_per_eval
# Create model
model_creator = get_model_creator(hparams)
train_model = model_helper.create_train_model(model_creator, hparams,
scope)
eval_model = model_helper.create_eval_model(model_creator, hparams, scope)
infer_model = model_helper.create_infer_model(model_creator, hparams,
scope)
# Preload data for sample decoding.
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
sample_src_data = inference.load_data(dev_src_file)
sample_tgt_data = inference.load_data(dev_tgt_file)
summary_name = "train_log"
model_dir = hparams.out_dir
# Log and output files
log_file = os.path.join(out_dir, "log_%d" % time.time())
log_f = tf.gfile.GFile(log_file, mode="a")
utils.print_out("# log_file=%s" % log_file, log_f)
# TensorFlow model
config_proto = utils.get_config_proto(
log_device_placement=log_device_placement,
num_intra_threads=hparams.num_intra_threads,
num_inter_threads=hparams.num_inter_threads)
train_sess = tf.Session(target=target_session,
config=config_proto,
graph=train_model.graph)
eval_sess = tf.Session(target=target_session,
config=config_proto,
graph=eval_model.graph)
infer_sess = tf.Session(target=target_session,
config=config_proto,
graph=infer_model.graph)
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
# Summary writer
summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name),
train_model.graph)
# First evaluation
run_full_eval(model_dir, infer_model, infer_sess, eval_model, eval_sess,
hparams, summary_writer, sample_src_data, sample_tgt_data,
avg_ckpts)
last_stats_step = global_step
last_eval_step = global_step
last_external_eval_step = global_step
# This is the training loop.
stats, info, start_train_time = before_train(loaded_train_model,
train_model, train_sess,
global_step, hparams, log_f)
while global_step < num_train_steps:
### Run a step ###
start_time = time.time()
try:
step_result = loaded_train_model.train(train_sess)
hparams.epoch_step += 1
except tf.errors.OutOfRangeError:
# Finished going through the training dataset. Go to next epoch.
hparams.epoch_step = 0
utils.print_out(
"# Finished an epoch, step %d. Perform external evaluation" %
global_step)
run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, sample_src_data, sample_tgt_data)
run_external_eval(infer_model, infer_sess, model_dir, hparams,
summary_writer)
if avg_ckpts:
run_avg_external_eval(infer_model, infer_sess, model_dir,
hparams, summary_writer, global_step)
train_sess.run(train_model.iterator.initializer,
feed_dict={train_model.skip_count_placeholder: 0})
continue
# Process step_result, accumulate stats, and write summary
global_step, info["learning_rate"], step_summary = update_stats(
stats, start_time, step_result)
summary_writer.add_summary(step_summary, global_step)
# Once in a while, we print statistics.
if global_step - last_stats_step >= steps_per_stats:
last_stats_step = global_step
is_overflow = process_stats(stats, info, global_step,
steps_per_stats, log_f)
print_step_info(" ", global_step, info, get_best_results(hparams),
log_f)
if is_overflow:
break
# Reset statistics
stats = init_stats()
if global_step - last_eval_step >= steps_per_eval:
last_eval_step = global_step
utils.print_out("# Save eval, global step %d" % global_step)
add_info_summaries(summary_writer, global_step, info)
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"translate.ckpt"),
global_step=global_step)
# Evaluate on dev/test
run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, sample_src_data, sample_tgt_data)
run_internal_eval(eval_model, eval_sess, model_dir, hparams,
summary_writer)
if global_step - last_external_eval_step >= steps_per_external_eval:
last_external_eval_step = global_step
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"translate.ckpt"),
global_step=global_step)
run_sample_decode(infer_model, infer_sess, model_dir, hparams,
summary_writer, sample_src_data, sample_tgt_data)
run_external_eval(infer_model, infer_sess, model_dir, hparams,
summary_writer)
if avg_ckpts:
run_avg_external_eval(infer_model, infer_sess, model_dir,
hparams, summary_writer, global_step)
# Done training
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir, "translate.ckpt"),
global_step=global_step)
(result_summary, _, final_eval_metrics) = (run_full_eval(
model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams,
summary_writer, sample_src_data, sample_tgt_data, avg_ckpts))
print_step_info("# Final, ", global_step, info, result_summary, log_f)
utils.print_time("# Done training!", start_train_time)
summary_writer.close()
utils.print_out("# Start evaluating saved best models.")
for metric in hparams.metrics:
best_model_dir = getattr(hparams, "best_" + metric + "_dir")
summary_writer = tf.summary.FileWriter(
os.path.join(best_model_dir, summary_name), infer_model.graph)
result_summary, best_global_step, _ = run_full_eval(
best_model_dir, infer_model, infer_sess, eval_model, eval_sess,
hparams, summary_writer, sample_src_data, sample_tgt_data)
print_step_info("# Best %s, " % metric, best_global_step, info,
result_summary, log_f)
summary_writer.close()
if avg_ckpts:
best_model_dir = getattr(hparams, "avg_best_" + metric + "_dir")
summary_writer = tf.summary.FileWriter(
os.path.join(best_model_dir, summary_name), infer_model.graph)
result_summary, best_global_step, _ = run_full_eval(
best_model_dir, infer_model, infer_sess, eval_model, eval_sess,
hparams, summary_writer, sample_src_data, sample_tgt_data)
print_step_info("# Averaged Best %s, " % metric, best_global_step,
info, result_summary, log_f)
summary_writer.close()
return final_eval_metrics, global_step
def run_external_eval(infer_model,
infer_sess,
model_dir,
hparams,
summary_writer,
save_best_dev=True,
use_test_set=True,
avg_ckpts=False,
dev_infer_iterator_feed_dict=None,
test_infer_iterator_feed_dict=None):
"""Compute external evaluation for both dev / test.
Computes development and testing external evaluation (e.g. bleu, rouge) for
given model.
Args:
infer_model: Inference model for which to compute perplexities.
infer_sess: Inference TensorFlow session.
model_dir: Directory from which to load inference model from.
hparams: Model hyper-parameters.
summary_writer: Summary writer for logging metrics to TensorBoard.
use_test_set: Computes testing external evaluation if true; does not
otherwise. Note that the development external evaluation is always
computed regardless of value of this parameter.
dev_infer_iterator_feed_dict: Feed dictionary for a TensorFlow session.
Can be used to pass in additional inputs necessary for running the
development external evaluation.
test_infer_iterator_feed_dict: Feed dictionary for a TensorFlow session.
Can be used to pass in additional inputs necessary for running the
testing external evaluation.
Returns:
Triple containing development scores, testing scores and the TensorFlow
Variable for the global step number, in this order.
"""
if dev_infer_iterator_feed_dict is None:
dev_infer_iterator_feed_dict = {}
if test_infer_iterator_feed_dict is None:
test_infer_iterator_feed_dict = {}
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src)
dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt)
dev_infer_iterator_feed_dict[
infer_model.src_placeholder] = inference.load_data(dev_src_file)
dev_infer_iterator_feed_dict[
infer_model.batch_size_placeholder] = hparams.infer_batch_size
dev_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
dev_infer_iterator_feed_dict,
dev_tgt_file,
"dev",
summary_writer,
save_on_best=save_best_dev,
avg_ckpts=avg_ckpts)
test_scores = None
if use_test_set and hparams.test_prefix:
test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src)
test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
test_infer_iterator_feed_dict[
infer_model.src_placeholder] = inference.load_data(test_src_file)
test_infer_iterator_feed_dict[
infer_model.batch_size_placeholder] = hparams.infer_batch_size
test_scores = _external_eval(loaded_infer_model,
global_step,
infer_sess,
hparams,
infer_model.iterator,
test_infer_iterator_feed_dict,
test_tgt_file,
"test",
summary_writer,
save_on_best=False,
avg_ckpts=avg_ckpts)
return dev_scores, test_scores, global_step
def train(hparams, identity, scope=None, target_session=""):
"""main loop to train the dialogue model. identity is used."""
out_dir = hparams.out_dir
steps_per_stats = hparams.steps_per_stats
steps_per_internal_eval = 3 * steps_per_stats
model_creator = diag_model.Model
train_model = model_helper.create_train_model(model_creator, hparams, scope)
model_dir = hparams.out_dir
# Log and output files
log_file = os.path.join(out_dir, identity+"log_%d" % time.time())
log_f = tf.gfile.GFile(log_file, mode="a")
utils.print_out("# log_file=%s" % log_file, log_f)
avg_step_time = 0.0
# load TensorFlow session and model
config_proto = utils.get_config_proto(
log_device_placement=hparams.log_device_placement,
allow_soft_placement=True)
train_sess = tf.Session(
target=target_session, config=config_proto, graph=train_model.graph)
train_handle = train_sess.run(train_model.train_iterator.string_handle())
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
# initialize summary writer
summary_writer = tf.summary.FileWriter(
os.path.join(out_dir, "train_log"), train_model.graph)
last_stats_step = global_step
last_eval_step = global_step
# initialize training stats.
step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0
checkpoint_total_count = 0.0
speed, train_ppl = 0.0, 0.0
start_train_time = time.time()
utils.print_out(
"# Start step %d, lr %g, %s" %
(global_step, loaded_train_model.learning_rate.eval(session=train_sess),
time.ctime()),
log_f)
# initialize iterators
skip_count = hparams.batch_size * hparams.epoch_step
utils.print_out("# Init train iterator, skipping %d elements" % skip_count)
train_sess.run(
train_model.train_iterator.initializer,
feed_dict={train_model.skip_count_placeholder: skip_count})
# main training loop
while global_step < hparams.num_train_steps:
start_time = time.time()
try: # run a step
step_result = loaded_train_model.train(train_sess, train_handle)
(_, step_loss, all_summaries, step_predict_count, step_summary,
global_step, step_word_count, batch_size, _, _, words1, words2, mask1,
mask2) = step_result
hparams.epoch_step += 1
except tf.errors.OutOfRangeError: # finished an epoch
hparams.epoch_step = 0
utils.print_out("# Finished an epoch, step %d." % global_step)
train_sess.run(
train_model.train_iterator.initializer,
feed_dict={train_model.skip_count_placeholder: 0})
continue
# Write step summary.
summary_writer.add_summary(step_summary, global_step)
for key in all_summaries:
utils.add_summary(summary_writer, global_step, key, all_summaries[key])
# update statistics
step_time += (time.time() - start_time)
checkpoint_loss += (step_loss * batch_size)
checkpoint_predict_count += step_predict_count
checkpoint_total_count += float(step_word_count)
if global_step - last_stats_step >= steps_per_stats:
# print statistics for the previous epoch and save the model.
last_stats_step = global_step
avg_step_time = step_time / steps_per_stats
utils.add_summary(summary_writer, global_step, "step_time", avg_step_time)
train_ppl = utils.safe_exp(checkpoint_loss / checkpoint_predict_count)
speed = checkpoint_total_count / (1000 * step_time)
if math.isnan(train_ppl):
break
# Reset timer and loss.
step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0
checkpoint_total_count = 0.0
# save the model
loaded_train_model.saver.save(
train_sess,
os.path.join(out_dir, "dialogue.ckpt"),
global_step=global_step)
# print the dialogue if in debug mode
if hparams.debug:
utils.print_current_dialogue(words1, words2, mask1, mask2)
# write out internal evaluation
if global_step - last_eval_step >= steps_per_internal_eval:
last_eval_step = global_step
utils.print_out("# Internal Evaluation. global step %d" % global_step)
utils.add_summary(summary_writer, global_step, "train_ppl", train_ppl)
# finished training
loaded_train_model.saver.save(
train_sess,
os.path.join(out_dir, "dialogue.ckpt"),
global_step=global_step)
result_summary = ""
utils.print_out(
"# Final, step %d lr %g "
"step-time %.2f wps %.2fK ppl %.2f, %s, %s" %
(global_step, loaded_train_model.learning_rate.eval(session=train_sess),
avg_step_time, speed, train_ppl, result_summary, time.ctime()),
log_f)
utils.print_time("# Done training!", start_train_time)
utils.print_out("# Start evaluating saved best models.")
summary_writer.close()
def train(flags):
"""Train the policy gradient model. """
out_dir = flags.out_dir
num_train_steps = flags.num_train_steps
steps_per_infer = flags.steps_per_infer
# Create model for train, infer mode
model_creator = get_model_creator(flags)
train_model = model_helper.create_train_model(flags, model_creator)
infer_model = model_helper.create_infer_model(flags, model_creator)
# TODO. set for distributed training and multi gpu
config_proto = tf.ConfigProto(allow_soft_placement=True)
config_proto.gpu_options.allow_growth = True
# Session for train, infer
train_sess = tf.Session(
config=config_proto, graph=train_model.graph)
infer_sess = tf.Session(
config=config_proto, graph=infer_model.graph)
# Load the train model if there's the file in the directory
# otherwise, initialize vars in the train model
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
out_dir, train_model.model, train_sess)
# Summary
train_summary = "train_log"
infer_summary = "infer_log"
# Summary writer for train, infer
train_summary_writer = tf.summary.FileWriter(
os.path.join(out_dir, train_summary), train_model.graph)
infer_summary_writer = tf.summary.FileWriter(
os.path.join(out_dir, infer_summary))
# First evaluation
run_infer(infer_model, out_dir, infer_sess)
# Initialize step var
last_infer_steps = global_step
# Training loop
while global_step < num_train_steps:
output_tuple = loaded_train_model.train(train_sess)
global_step = output_tuple.global_step
train_summary = output_tuple.train_summary
# Update train summary
train_summary_writer.add_summary(train_summary, global_step)
print('current global_step: {}'.format(global_step))
# Evaluate the model for steps_per_infer
if global_step - last_infer_steps >= steps_per_infer:
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir, "rl.ckpt"), global_step)
last_infer_steps = global_step
output_tuple = run_infer(infer_model, out_dir, infer_sess)
infer_summary = output_tuple.infer_summary
# Update infer summary
infer_summary_writer.add_summary(infer_summary, global_step)
# Done training
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir, "rl.ckpt"), global_step)
print('Train done')
def train(hps, scope=None, target_session=""):
"""Train a translation model."""
log_device_placement = hps.log_device_placement
out_dir = hps.out_dir
num_train_steps = hps.num_train_steps
steps_per_stats = hps.steps_per_stats
steps_per_external_eval = hps.steps_per_external_eval
steps_per_eval = 100 * steps_per_stats
if not steps_per_external_eval:
steps_per_external_eval = 5 * steps_per_eval
if hps.attention_architecture == "baseline":
model_creator = AttentionModel
else:
model_creator = AttentionHistoryModel
train_model = model_helper.create_train_model(model_creator, hps, scope)
eval_model = model_helper.create_eval_model(model_creator, hps, scope)
infer_model = model_helper.create_infer_model(model_creator, hps, scope)
# Preload data for sample decoding.
article_filenames = []
abstract_filenames = []
art_dir = hps.data_dir + '/article'
abs_dir = hps.data_dir + '/abstract'
for file in os.listdir(art_dir):
if file.startswith(hps.dev_prefix):
article_filenames.append(art_dir + "/" + file)
for file in os.listdir(abs_dir):
if file.startswith(hps.dev_prefix):
abstract_filenames.append(abs_dir + "/" + file)
# if random_decode:
# """if this is a random sampling process during training"""
decode_id = random.randint(0, len(article_filenames) - 1)
single_article_file = article_filenames[decode_id]
single_abstract_file = abstract_filenames[decode_id]
dev_src_file = single_article_file
dev_tgt_file = single_abstract_file
sample_src_data = inference_base_model.load_data(dev_src_file)
sample_tgt_data = inference_base_model.load_data(dev_tgt_file)
summary_name = "train_log"
model_dir = hps.out_dir
# Log and output files
log_file = os.path.join(out_dir, "log_%d" % time.time())
log_f = tf.gfile.GFile(log_file, mode="a")
utils.print_out("# log_file=%s" % log_file, log_f)
avg_step_time = 0.0
# TensorFlow model
config_proto = utils.get_config_proto(
log_device_placement=log_device_placement)
train_sess = tf.Session(target=target_session,
config=config_proto,
graph=train_model.graph)
eval_sess = tf.Session(target=target_session,
config=config_proto,
graph=eval_model.graph)
infer_sess = tf.Session(target=target_session,
config=config_proto,
graph=infer_model.graph)
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
# Summary writer
summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name),
train_model.graph)
# First evaluation
# run_full_eval(
# model_dir, infer_model, infer_sess,
# eval_model, eval_sess, hps,
# summary_writer,sample_src_data,sample_tgt_data)
last_stats_step = global_step
last_eval_step = global_step
last_external_eval_step = global_step
# This is the training loop.
stats = init_stats()
speed, train_ppl = 0.0, 0.0
start_train_time = time.time()
utils.print_out(
"# Start step %d, lr %g, %s" %
(global_step, loaded_train_model.learning_rate.eval(
session=train_sess), time.ctime()), log_f)
# Initialize all of the iterators
skip_count = hps.batch_size * hps.epoch_step
utils.print_out("# Init train iterator, skipping %d elements" % skip_count)
train_sess.run(train_model.iterator.initializer,
feed_dict={train_model.skip_count_placeholder: skip_count})
epoch_step = 0
while global_step < num_train_steps:
### Run a step ###
start_time = time.time()
try:
step_result = loaded_train_model.train(train_sess)
epoch_step += 1
except tf.errors.OutOfRangeError:
# Finished going through the training dataset. Go to next epoch.
epoch_step = 0
utils.print_out(
"# Finished an epoch, step %d. Perform external evaluation" %
global_step)
run_sample_decode(infer_model, infer_sess, model_dir, hps,
summary_writer, sample_src_data, sample_tgt_data)
dev_scores, test_scores, _ = run_external_eval(
infer_model, infer_sess, model_dir, hps, summary_writer)
train_sess.run(train_model.iterator.initializer,
feed_dict={train_model.skip_count_placeholder: 0})
continue
# Write step summary and accumulate statistics
global_step = update_stats(stats, summary_writer, start_time,
step_result)
# Once in a while, we print statistics.
if global_step - last_stats_step >= steps_per_stats:
last_stats_step = global_step
is_overflow = check_stats(stats, global_step, steps_per_stats, hps,
log_f)
if is_overflow:
break
# Reset statistics
stats = init_stats()
if global_step - last_eval_step >= steps_per_eval:
last_eval_step = global_step
utils.print_out("# Save eval, global step %d" % global_step)
utils.add_summary(summary_writer, global_step, "train_ppl",
train_ppl)
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"summarized.ckpt"),
global_step=global_step)
# Evaluate on dev/test
run_sample_decode(infer_model, infer_sess, model_dir, hps,
summary_writer, sample_src_data, sample_tgt_data)
dev_ppl, test_ppl = run_internal_eval(eval_model, eval_sess,
model_dir, hps,
summary_writer)
if global_step - last_external_eval_step >= steps_per_external_eval:
last_external_eval_step = global_step
# Save checkpoint
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir,
"summarized.ckpt"),
global_step=global_step)
run_sample_decode(infer_model, infer_sess, model_dir, hps,
summary_writer, sample_src_data, sample_tgt_data)
dev_scores, test_scores, _ = run_external_eval(
infer_model, infer_sess, model_dir, hps, summary_writer)
# Done training
loaded_train_model.saver.save(train_sess,
os.path.join(out_dir, "summarized.ckpt"),
global_step=global_step)
result_summary, _, dev_scores, test_scores, dev_ppl, test_ppl = run_full_eval(
model_dir, infer_model, infer_sess, eval_model, eval_sess, hps,
summary_writer, sample_src_data, sample_tgt_data)
utils.print_out(
"# Final, step %d lr %g "
"step-time %.2f wps %.2fK ppl %.2f, %s, %s" %
(global_step,
loaded_train_model.learning_rate.eval(session=train_sess),
avg_step_time, speed, train_ppl, result_summary, time.ctime()), log_f)
utils.print_time("# Done training!", start_train_time)
summary_writer.close()
utils.print_out("# Start evaluating saved best models.")
for metric in hps.metrics:
best_model_dir = getattr(hps, "best_" + metric + "_dir")
summary_writer = tf.summary.FileWriter(
os.path.join(best_model_dir, summary_name), infer_model.graph)
result_summary, best_global_step, _, _, _, _ = run_full_eval(
best_model_dir, infer_model, infer_sess, eval_model, eval_sess,
hps, summary_writer, sample_src_data, sample_tgt_data)
utils.print_out(
"# Best %s, step %d "
"step-time %.2f wps %.2fK, %s, %s" %
(metric, best_global_step, avg_step_time, speed, result_summary,
time.ctime()), log_f)
summary_writer.close()
return (dev_scores, test_scores, dev_ppl, test_ppl, global_step)
def train(hparams):
"""Train a seq2seq model."""
log_device_placement = hparams.log_device_placement
out_dir = hparams.out_dir
num_train_steps = hparams.num_train_steps
steps_per_stats = hparams.steps_per_stats
model_creator = model.Model
train_model = model_helper.create_train_model(model_creator, hparams)
summary_name = "train_log"
model_dir = hparams.out_dir
# Log and output files
log_file = os.path.join(out_dir, "log_%d" % time.time())
log_f = tf.gfile.GFile(log_file, mode="a")
utils.print_out("# log_files=%s" % log_file, log_f)
# TensorFlow model
config_proto = utils.get_config_proto(
log_device_placement=log_device_placement,
num_intra_threads=hparams.num_intra_threads,
num_inter_threads=hparams.num_inter_threads)
train_sess = tf.Session(config=config_proto, graph=train_model.graph)
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
# Summary writer
summary_writer = tf.summary.FileWriter(
os.path.join(out_dir, summary_name), train_model.graph)
last_stats_step = global_step
# This is the training loop.
stats, info, start_train_time = before_train(
loaded_train_model, train_model, train_sess, global_step, hparams, log_f)
while global_step < num_train_steps:
### Run a step ###
start_time = time.time()
try:
step_result = loaded_train_model.train(train_sess)
hparams.epoch_step += 1
except tf.errors.OutOfRangeError:
# Finished going through the training dataset. Go to next epoch.
hparams.epoch_step = 0
utils.print_out("# Finished an epoch, step %d." % global_step)
train_sess.run(
train_model.iterator.initializer,
feed_dict={train_model.skip_count_placeholder: 0})
continue
# Process step_result, accumulate stats, and write summary
global_step, info["learning_rate"], step_summary = update_stats(
stats, start_time, step_result)
summary_writer.add_summary(step_summary, global_step)
# Once in a while, we print statistics.
if global_step - last_stats_step >= steps_per_stats:
last_stats_step = global_step
is_overflow = process_stats(stats, info, global_step, steps_per_stats, log_f)
print_step_info(" ", global_step, info, log_f)
if is_overflow:
break
# Reset statistics
stats = init_stats()
# Done training
loaded_train_model.saver.save(
train_sess,
os.path.join(out_dir, "seq2seq.ckpt"),
global_step=global_step)
summary_writer.close()
return global_step
def self_play_eval_fn(hparams,
identity,
num_workers=1,
jobid=0,
scope=None,
target_session=''):
"""This is the single worker self play.
Mostly used for self play
evaluation. identity is used here to distinguish between workers.
"""
model_creator = diag_model.Model
mutable_model = model_helper.create_selfplay_model(
model_creator,
True, # mutable is True
num_workers,
jobid,
hparams=hparams,
scope=scope)
immutable_model = model_helper.create_selfplay_model(
model_creator,
False, # mutable is False
num_workers,
jobid,
hparams=hparams,
scope=scope)
mutable_sess = tf.Session(graph=mutable_model.graph,
config=tf.ConfigProto(
allow_soft_placement=True,
device_count={'GPU': hparams.num_gpus}))
immutable_sess = tf.Session(graph=immutable_model.graph,
config=tf.ConfigProto(
allow_soft_placement=True,
device_count={'GPU': hparams.num_gpus}))
# number of steps per external eval
steps_per_external_eval = 10
# force conducting a self play at the beginning
last_external_eval_step = -1 * steps_per_external_eval
print('hparams.self_play_pretrain_dir=', hparams.self_play_pretrain_dir)
print('steps_per_external_eval=', steps_per_external_eval)
writer_path = os.path.join(hparams.out_dir,
identity + hparams.task_type + '_log')
summary_writer = tf.summary.FileWriter(writer_path, mutable_sess.graph)
print('summary_writer estabilished at', writer_path)
# waiting for checkpoints and loop forever
latest_ckpt = None
while True:
latest_ckpt = tf.contrib.training.wait_for_new_checkpoint(
hparams.out_dir, latest_ckpt)
print('got checkpoint', latest_ckpt)
# get the global_step variable first
with mutable_model.graph.as_default():
# first initialize to avoid encountering missing component for adam optimizer
_, global_step = model_helper.create_or_load_model(
mutable_model.model, hparams.out_dir, mutable_sess,
hparams.task_type)
# valid evaluation step
if (not hparams.eval_forever) or (global_step - last_external_eval_step
>= steps_per_external_eval):
# if eval_forever is disabled, we will do one selfplay evalation
# otherwise, we will wait until certain number of timesteps are elapsed.
last_external_eval_step = global_step
print('do single worker evaluation')
single_worker_selfplay(mutable_model, immutable_model,
mutable_sess, immutable_sess,
hparams.self_play_eval_data,
hparams.self_play_eval_kb, global_step,
hparams, summary_writer)
else:
print('Wait until steps_per_external_eval is reached.',
global_step, last_external_eval_step,
steps_per_external_eval)
if not hparams.eval_forever:
break # if eval_foever is disabled, we only evaluate once
mutable_sess.close()
immutable_sess.close()
