def secondary_fn_tmp(hparams, identity, model_dir, model, eval_model, eval_sess, name, worker_fn): """secondary helper function for inference and evaluation.""" steps_per_external_eval = 10 # initialize summary writer summary_writer_path = os.path.join(hparams.out_dir, identity + name + "_log") print("summary_writer_path", summary_writer_path) summary_writer = tf.summary.FileWriter(summary_writer_path, model.graph) config_proto = utils.get_config_proto( log_device_placement=hparams.log_device_placement, allow_soft_placement=True) # create session sess = tf.Session(config=config_proto, graph=model.graph) # wait for the checkpoints latest_ckpt = None last_external_eval_step = 0 # main inference loop while True: latest_ckpt = tf.contrib.training.wait_for_new_checkpoint( model_dir, latest_ckpt) with model.graph.as_default(): _, global_step = model_helper.create_or_load_model( model.model, model_dir, sess, name) if global_step - last_external_eval_step >= steps_per_external_eval: last_external_eval_step = global_step worker_fn(model, sess, eval_model, eval_sess, latest_ckpt, summary_writer, global_step, hparams) if not hparams.eval_forever: break # if eval_foever is disabled, we only evaluate once summary_writer.close() sess.close()
def eval(task, test_data): data_seq, data_label0, data_label1 = gen_data(test_data, mode='train') data_len = len(data_seq) eval_step = int(math.ceil(float(data_len) / _hp.batch_size)) data_eval = [data_seq, data_label0, data_label1] with tf.Graph().as_default() as graph: with tf.Session(graph=graph) as sess: sess.run(tf.global_variables_initializer()) model = Model(task, 'test') model, global_step = _mh.create_or_load_model( model, _hp.model_path, sess) eval_tq = tqdm(range(eval_step)) l_total = 0.0 for eval_step_it in eval_tq: batch = gen_train_data(data_eval, _hp.batch_size, eval_step_it) if len(batch) == 0: break batch_data = data2ids(batch) data_sess = batch_data[0] subject_label = batch_data[1] sentiment_label = batch_data[2] label = model.model_exe(sess, "eval", data_sess) label_result = None if task == 'subject': label_result = subject_label else: label_result = sentiment_label acc_rate = eval_process(label_result, label) eval_tq.set_description("%s acc: %f" % (task, acc_rate)) l_total = l_total + acc_rate print("%s train set average acc: %f" % (task, l_total / float(eval_step)))
def run_internal_eval(eval_model, eval_sess, model_dir, hps, summary_writer, use_test_set=False): """Compute internal evaluation (perplexity) for both dev / test.""" with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval") dev_src_files, dev_tgt_files = data.get_files(hps.data_dir, hps.test_prefix) dev_eval_iterator_feed_dict = { eval_model.src_file_placeholder: dev_src_files, eval_model.tgt_file_placeholder: dev_tgt_files } dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, dev_eval_iterator_feed_dict, summary_writer, "dev") test_ppl = None if use_test_set and hps.test_prefix: test_src_files, test_tgt_files = data.get_files( hps.data_dir, hps.test_prefix) test_eval_iterator_feed_dict = { eval_model.src_file_placeholder: test_src_files, eval_model.tgt_file_placeholder: test_tgt_files } test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, test_eval_iterator_feed_dict, summary_writer, "test") return dev_ppl, test_ppl
def run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, save_best_dev=True, use_test_set=True, avg_ckpts=False): """ Compute external evaluation (bleu, rouge, etc.) for both dev / test. """ with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") dev_scores = None test_scores = None if global_step > 0: utils.log("External evaluation, global step %d" % global_step) dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_infer_iterator_feed_dict = { infer_model.src_placeholder: utils.load_data(dev_src_file), infer_model.batch_size_placeholder: hparams.infer_batch_size, } dev_scores = external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, dev_infer_iterator_feed_dict, dev_tgt_file, "dev", summary_writer, save_on_best=save_best_dev, avg_ckpts=avg_ckpts) if use_test_set and hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_infer_iterator_feed_dict = { infer_model.src_placeholder: utils.load_data(test_src_file), infer_model.batch_size_placeholder: hparams.infer_batch_size, } test_scores = external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, test_infer_iterator_feed_dict, test_tgt_file, "test", summary_writer, save_on_best=False, avg_ckpts=avg_ckpts) return dev_scores, test_scores, global_step
def run_sample_decode(args, infer_model, infer_sess, model_dir, infer_data): with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") _sample_decode(args, loaded_infer_model, global_step, infer_sess, infer_data)
def run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer, use_test_set=True, ckpt_index=None): """Compute internal evaluation (perplexity) for both dev / test.""" with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval", ckpt_index) dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_eval_iterator_feed_dict = { eval_model.src_file_placeholder: dev_src_file, eval_model.tgt_file_placeholder: dev_tgt_file } dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, dev_eval_iterator_feed_dict, summary_writer, "dev") test_ppl = None if use_test_set and hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_eval_iterator_feed_dict = { eval_model.src_file_placeholder: test_src_file, eval_model.tgt_file_placeholder: test_tgt_file } test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, test_eval_iterator_feed_dict, summary_writer, "test") return dev_ppl, test_ppl
def run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer, use_test_set=True, dev_eval_iterator_feed_dict=None, test_eval_iterator_feed_dict=None): """Compute internal evaluation (perplexity) for both dev / test. Computes development and testing perplexities for given model. Args: eval_model: Evaluation model for which to compute perplexities. eval_sess: Evaluation TensorFlow session. model_dir: Directory from which to load evaluation model from. hparams: Model hyper-parameters. summary_writer: Summary writer for logging metrics to TensorBoard. use_test_set: Computes testing perplexity if true; does not otherwise. Note that the development perplexity is always computed regardless of value of this parameter. dev_eval_iterator_feed_dict: Feed dictionary for a TensorFlow session. Can be used to pass in additional inputs necessary for running the development evaluation. test_eval_iterator_feed_dict: Feed dictionary for a TensorFlow session. Can be used to pass in additional inputs necessary for running the testing evaluation. Returns: Pair containing development perplexity and testing perplexity, in this order. """ if dev_eval_iterator_feed_dict is None: dev_eval_iterator_feed_dict = {} if test_eval_iterator_feed_dict is None: test_eval_iterator_feed_dict = {} with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval") dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_eval_iterator_feed_dict[eval_model.src_file_placeholder] = dev_src_file dev_eval_iterator_feed_dict[eval_model.tgt_file_placeholder] = dev_tgt_file dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, dev_eval_iterator_feed_dict, summary_writer, "dev") test_ppl = None if use_test_set and hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_eval_iterator_feed_dict[ eval_model.src_file_placeholder] = test_src_file test_eval_iterator_feed_dict[ eval_model.tgt_file_placeholder] = test_tgt_file test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, test_eval_iterator_feed_dict, summary_writer, "test") return dev_ppl, test_ppl
def start_sess_and_load_model(infer_model, ckpt_dir): sess = tf.Session( graph=infer_model.graph, config=misc.get_config_proto()) with infer_model.graph.as_default(): loaded_infer_model, global_step = helper.create_or_load_model( infer_model.model, ckpt_dir, sess) return sess, loaded_infer_model, global_step
def run_infer(infer_model, model_dir, infer_sess): """Load vars. then, run infer model for 1 episode.""" with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( model_dir, infer_model.model, infer_sess) output_tuple = loaded_infer_model.infer(infer_sess) return output_tuple
def run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, src_data, tgt_data): """Sample decode a random sentence from src_data.""" with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") _sample_decode(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, src_data, tgt_data, infer_model.src_placeholder, summary_writer)
def train_fn(hparams): graph, model, iterator = create_train_model(hparams) sess = tf.Session(graph=graph) with graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( model, hparams.out_dir, sess, "train") sess.run(model.iterator.initializer) while global_step < hparams.num_train_steps: start_time = time.time() _, train_loss, train_accuracy, train_summary, global_step, _, _, _ = loaded_train_model.train( sess) print(train_loss, train_accuracy)
def run_external_eval(infer_model, infer_sess, model_dir, hps, summary_writer, save_best_dev=True, use_test_set=True): """Compute external evaluation (bleu, rouge, etc.) for both dev / test.""" with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") dev_src_file = hps.dev_src_file dev_tgt_file = hps.dev_tgt_file dev_infer_iterator_feed_dict = { infer_model.src_placeholder: inference_base_model.load_data(dev_src_file), infer_model.batch_size_placeholder: hps.infer_batch_size, } dev_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hps, infer_model.iterator, dev_infer_iterator_feed_dict, dev_tgt_file, "dev", summary_writer, save_on_best=save_best_dev) test_scores = None if use_test_set and hps.test_prefix: test_src_file = hps.test_src_file test_tgt_file = hps.test_tgt_file test_infer_iterator_feed_dict = { infer_model.src_placeholder: inference_base_model.load_data(test_src_file), infer_model.batch_size_placeholder: hps.infer_batch_size, } test_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hps, infer_model.iterator, test_infer_iterator_feed_dict, test_tgt_file, "test", summary_writer, save_on_best=False) return dev_scores, test_scores, global_step
def run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, save_best_dev=True): """Compute external evaluation (bleu, rouge, etc.) for both dev / test.""" with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_infer_iterator_feed_dict = { infer_model.src_placeholder: inference.load_data(dev_src_file) } dev_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, dev_infer_iterator_feed_dict, dev_tgt_file, "dev", summary_writer, save_on_best=save_best_dev) test_scores = None if hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_infer_iterator_feed_dict = { infer_model.src_placeholder: inference.load_data(test_src_file) } test_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, test_infer_iterator_feed_dict, test_tgt_file, "test", summary_writer, save_on_best=False) return dev_scores, test_scores, global_step
def run_evaluation(eval_model, eval_sess, model_dir, input_eval_file, output_eval_file, input_emb_weights, summary_writer): with eval_model.graph.as_default(): # initialize the variables of the eval graph in eval_sess or load them from a checkpoint. loaded_eval_model = model_helper.create_or_load_model( eval_model.model, eval_sess, "eval", model_dir, input_emb_weights) eval_iterator_feed_dict = { eval_model.input_file_placeholder: input_eval_file, eval_model.output_file_placeholder: output_eval_file } dev_loss = evaluation.eval(loaded_eval_model, eval_sess, eval_model.iterator, eval_iterator_feed_dict) model_helper.add_summary(summary_writer, "dev_loss", dev_loss) return dev_loss
def run_internal_eval(eval_model, eval_handle, eval_sess, model_dir, hparams, summary_writer): """Compute internal evaluation (perplexity) for dev.""" with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval") dev_eval_iterator_feed_dict = { eval_model.data_file_placeholder: hparams.dev_data, eval_model.kb_file_placeholder: hparams.dev_kb, } dev_ppl = _internal_eval( loaded_eval_model, global_step, eval_sess, eval_model.eval_iterator, dev_eval_iterator_feed_dict, eval_handle, summary_writer, "dev") return dev_ppl, None
def _createTestInferCheckpoint(self, hparams, out_dir): if not hparams.attention: model_creator = nmt_model.Model elif hparams.attention_architecture == "standard": model_creator = attention_model.AttentionModel elif hparams.attention_architecture in ["gnmt", "gnmt_v2"]: model_creator = gnmt_model.GNMTModel else: raise ValueError("Unknown model architecture") infer_model = model_helper.create_infer_model(model_creator, hparams) with self.test_session(graph=infer_model.graph) as sess: loaded_model, global_step = model_helper.create_or_load_model( infer_model.model, out_dir, sess, "infer_name") ckpt = loaded_model.saver.save(sess, os.path.join( out_dir, "translate.ckpt"), global_step=global_step) return ckpt
def infer(hparams, infer_model, infer_sess, batch_size=1): src1_vocab_file = "%s/%s" % (hparams.data_dir, hparams.word_vocab) src2_vocab_file = "%s/%s" % (hparams.data_dir, hparams.pos_vocab) tgt_vocab_file = "%s/%s" % (hparams.data_dir, hparams.role_vocab) with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, hparams.ckpt_dir, infer_sess, "infer") test_src1_file = "%s/%s" % (hparams.data_dir, hparams.test_word_data) test_src2_file = "%s/%s" % (hparams.data_dir, hparams.test_pos_data) test_src1_data = load_data(test_src1_file) test_src2_data = load_data(test_src2_file) test_tgt_file = "%s/%s_%d" % (hparams.output_dir, hparams.test_role_data, global_step) test_infer_iterator_feed_dict = { infer_model.src1_placeholder: test_src1_data, infer_model.src2_placeholder: test_src2_data, infer_model.batch_size_placeholder: batch_size } print(" start infer..") infer_sess.run(infer_model.iterator.initializer, feed_dict=test_infer_iterator_feed_dict) outfile = open(test_tgt_file, "w", encoding="utf-8") infer_num = 200 while True and infer_num > 0: try: predict = loaded_infer_model.infer(infer_sess) from_vocab, rev_from_vocab = data_utils.initialize_vocabulary( src1_vocab_file) _, rev_to_vocab = data_utils.initialize_vocabulary( tgt_vocab_file) sample_output = predict[0].tolist() outputs = [] # alignment = alignment[0].tolist() for output in sample_output: outputs.append(tf.compat.as_str(rev_to_vocab[output])) #print(" ".join(outputs)) outfile.write(" ".join(outputs) + "\n") infer_num -= 1 except tf.errors.OutOfRangeError: break print(" infer done.") outfile.close()
def _createTestInferCheckpoint(self, hparams, name): # Prepare hparams.vocab_prefix = ("nmt/testdata/test_infer_vocab") hparams.src_vocab_file = hparams.vocab_prefix + "." + hparams.src hparams.tgt_vocab_file = hparams.vocab_prefix + "." + hparams.tgt out_dir = os.path.join(tf.test.get_temp_dir(), name) os.makedirs(out_dir) hparams.out_dir = out_dir # Create check point model_creator = inference.get_model_creator(hparams) infer_model = model_helper.create_infer_model(model_creator, hparams) with self.test_session(graph=infer_model.graph) as sess: loaded_model, global_step = model_helper.create_or_load_model( infer_model.model, out_dir, sess, "infer_name") ckpt_path = loaded_model.saver.save(sess, os.path.join( out_dir, "translate.ckpt"), global_step=global_step) return ckpt_path
def run_internal_eval(args, eval_model, eval_sess, model_dir, eval_data): with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval") eval_data_size = len(eval_data) eval_data = utils.get_batches(eval_data, args.max_batch) eval_total_loss = 0.0 total_predict_count = 0.0 for idx, batch in enumerate(eval_data): eval_loss, predict_count, batch_size = loaded_eval_model.eval( eval_sess, batch.user, batch.product, batch.rating, batch.review_input, batch.review_output, batch.review_length) eval_total_loss += eval_loss * batch_size total_predict_count += predict_count eval_avg_loss = eval_total_loss / eval_data_size eval_ppl = np.exp(eval_total_loss / total_predict_count) return eval_avg_loss, eval_ppl
def run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer, use_test_set=True): """Compute internal evaluation (perplexity) for both dev / test.""" with eval_model.graph.as_default(): loaded_eval_model, global_step = model_helper.create_or_load_model( eval_model.model, model_dir, eval_sess, "eval") dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_eval_iterator_feed_dict = { eval_model.src_file_placeholder: dev_src_file, eval_model.tgt_file_placeholder: dev_tgt_file } dev_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, dev_eval_iterator_feed_dict, summary_writer, "dev") test_ppl = None if use_test_set and hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_eval_iterator_feed_dict = { eval_model.src_file_placeholder: test_src_file, eval_model.tgt_file_placeholder: test_tgt_file } test_ppl = _internal_eval(loaded_eval_model, global_step, eval_sess, eval_model.iterator, test_eval_iterator_feed_dict, summary_writer, "test") with open('forward_ppl.txt', 'a+') as fw: fw.write('dev_ppl: ') fw.write(str(round(dev_ppl, 2))) fw.write('\ttest_ppl: ') fw.write(str(round(test_ppl, 2)) + '\n') return dev_ppl, test_ppl
def inference(task): datas = get_data_list(mode='test') test_data = gen_data(datas, mode='test') test_len = len(test_data) test_step = int(math.ceil(float(test_len) / _hp.batch_size)) with tf.Graph().as_default() as graph: with tf.Session(graph=graph) as sess: sess.run(tf.global_variables_initializer()) model = Model(task, 'test') model, global_step = _mh.create_or_load_model( model, _hp.model_path, sess) label_out = [] for step in tqdm(range(test_step)): data_test = gen_inference_data(test_data, _hp.batch_size, step) data_test_ids = data2ids(data_test, mode='test') label = model.model_exe(sess, "test", data_test_ids) label = [label.values, label.indices] # TopKV2 to list l = id2label(label, task, type='test') label_out = label_out + l csv_write(datas, label_out, task) tf.train.write_graph(sess.graph_def, _hp.model_path, 'sentiment.pbtxt')
def run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, src_data, tgt_data): """ Sample decode a random sentence from src_data. """ with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") # Pick a sentence and decode.""" decode_id = random.randint(0, len(src_data) - 1) iterator_feed_dict = { infer_model.src_placeholder: [src_data[decode_id]], infer_model.batch_size_placeholder: 1, } infer_sess.run(infer_model.iterator.initializer, feed_dict=iterator_feed_dict) nmt_outputs, attention_summary = loaded_infer_model.decode(infer_sess) if hparams.beam_width > 0: # get the top translation. nmt_outputs = nmt_outputs[0] translation = utils.get_translation(nmt_outputs, sent_id=0, tgt_eos=hparams.eos) utils.log("Sample src: {}".format(src_data[decode_id])) utils.log("Sample ref: {}".format(tgt_data[decode_id])) utils.log("NMT output: {}".format(translation)) # Summary if attention_summary is not None: summary_writer.add_summary(attention_summary, global_step)
def train(hparams, scope=None, target_session="", compute_ppl=0): """Train a translation model.""" log_device_placement = hparams.log_device_placement out_dir = hparams.out_dir num_train_steps = hparams.num_train_steps steps_per_stats = hparams.steps_per_stats steps_per_external_eval = hparams.steps_per_external_eval steps_per_eval = 10 * steps_per_stats avg_ckpts = hparams.avg_ckpts if not steps_per_external_eval: steps_per_external_eval = 5 * steps_per_eval if not hparams.attention: # choose this model model_creator = nmt_model.Model else: # Attention if (hparams.encoder_type == "gnmt" or hparams.attention_architecture in ["gnmt", "gnmt_v2"]): model_creator = gnmt_model.GNMTModel elif hparams.attention_architecture == "standard": model_creator = attention_model.AttentionModel else: raise ValueError("Unknown attention architecture %s" % hparams.attention_architecture) train_model = model_helper.create_train_model(model_creator, hparams, scope) eval_model = model_helper.create_eval_model(model_creator, hparams, scope) infer_model = model_helper.create_infer_model(model_creator, hparams, scope) # Preload data for sample decoding. dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) sample_src_data = inference.load_data(dev_src_file) sample_tgt_data = inference.load_data(dev_tgt_file) wsd_src_file = "%s" % (hparams.sample_prefix) wsd_src_data = inference.load_data(wsd_src_file) summary_name = "train_log" model_dir = hparams.out_dir # Log and output files log_file = os.path.join(out_dir, "log_%d" % time.time()) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) # TensorFlow model config_proto = utils.get_config_proto( log_device_placement=log_device_placement, num_intra_threads=hparams.num_intra_threads, num_inter_threads=hparams.num_inter_threads) train_sess = tf.Session(target=target_session, config=config_proto, graph=train_model.graph) eval_sess = tf.Session(target=target_session, config=config_proto, graph=eval_model.graph) infer_sess = tf.Session(target=target_session, config=config_proto, graph=infer_model.graph) with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( train_model.model, model_dir, train_sess, "train") # Summary writer summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name), train_model.graph) # First evaluation ''' run_full_eval( model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams, summary_writer, sample_src_data, sample_tgt_data, avg_ckpts) ''' last_stats_step = global_step last_eval_step = global_step last_external_eval_step = global_step # This is the training loop. stats, info, start_train_time = before_train(loaded_train_model, train_model, train_sess, global_step, hparams, log_f) end_step = global_step + 100 while global_step < end_step: # num_train_steps ### Run a step ### start_time = time.time() try: # then forward inference result to WSD, get reward step_result = loaded_train_model.train(train_sess) # forward reward to placeholder of loaded_train_model, and write a new train function where loss = loss*reward hparams.epoch_step += 1 except tf.errors.OutOfRangeError: # Finished going through the training dataset. Go to next epoch. hparams.epoch_step = 0 utils.print_out( "# Finished an epoch, step %d. Perform external evaluation" % global_step) # run_sample_decode(infer_model, infer_sess, model_dir, hparams, # summary_writer, sample_src_data, sample_tgt_data) # only for pretrain # run_external_eval(infer_model, infer_sess, model_dir, hparams, # summary_writer) if avg_ckpts: run_avg_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, global_step) train_sess.run(train_model.iterator.initializer, feed_dict={train_model.skip_count_placeholder: 0}) continue # Process step_result, accumulate stats, and write summary global_step, info["learning_rate"], step_summary = update_stats( stats, start_time, step_result, hparams) summary_writer.add_summary(step_summary, global_step) if compute_ppl: run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer) # Once in a while, we print statistics. if global_step - last_stats_step >= steps_per_stats: last_stats_step = global_step is_overflow = process_stats(stats, info, global_step, steps_per_stats, log_f) print_step_info(" ", global_step, info, _get_best_results(hparams), log_f) if is_overflow: break # Reset statistics stats = init_stats() if global_step - last_eval_step >= steps_per_eval: last_eval_step = global_step utils.print_out("# Save eval, global step %d" % global_step) utils.add_summary(summary_writer, global_step, "train_ppl", info["train_ppl"]) # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) # Evaluate on dev/test run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, sample_src_data, sample_tgt_data) run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer) if global_step - last_external_eval_step >= steps_per_external_eval: last_external_eval_step = global_step # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, sample_src_data, sample_tgt_data) run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer) if avg_ckpts: run_avg_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, global_step) # Done training loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) '''
def train(hparams, scope=None, target_session=""): """Train a translation model.""" log_device_placement = hparams.log_device_placement out_dir = hparams.out_dir num_train_steps = hparams.num_train_steps steps_per_stats = hparams.steps_per_stats steps_per_external_eval = hparams.steps_per_external_eval steps_per_eval = 10 * steps_per_stats avg_ckpts = hparams.avg_ckpts if not steps_per_external_eval: steps_per_external_eval = 5 * steps_per_eval # Create model model_creator = get_model_creator(hparams) train_model = model_helper.create_train_model(model_creator, hparams, scope) eval_model = model_helper.create_eval_model(model_creator, hparams, scope) infer_model = model_helper.create_infer_model(model_creator, hparams, scope) # Preload data for sample decoding. dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) sample_src_data = inference.load_data(dev_src_file) sample_tgt_data = inference.load_data(dev_tgt_file) summary_name = "train_log" model_dir = hparams.out_dir # Log and output files log_file = os.path.join(out_dir, "log_%d" % time.time()) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) # TensorFlow model config_proto = utils.get_config_proto( log_device_placement=log_device_placement, num_intra_threads=hparams.num_intra_threads, num_inter_threads=hparams.num_inter_threads) train_sess = tf.Session(target=target_session, config=config_proto, graph=train_model.graph) eval_sess = tf.Session(target=target_session, config=config_proto, graph=eval_model.graph) infer_sess = tf.Session(target=target_session, config=config_proto, graph=infer_model.graph) with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( train_model.model, model_dir, train_sess, "train") # Summary writer summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name), train_model.graph) # First evaluation run_full_eval(model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams, summary_writer, sample_src_data, sample_tgt_data, avg_ckpts) last_stats_step = global_step last_eval_step = global_step last_external_eval_step = global_step # This is the training loop. stats, info, start_train_time = before_train(loaded_train_model, train_model, train_sess, global_step, hparams, log_f) while global_step < num_train_steps: ### Run a step ### start_time = time.time() try: step_result = loaded_train_model.train(train_sess) hparams.epoch_step += 1 except tf.errors.OutOfRangeError: # Finished going through the training dataset. Go to next epoch. hparams.epoch_step = 0 utils.print_out( "# Finished an epoch, step %d. Perform external evaluation" % global_step) run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, sample_src_data, sample_tgt_data) run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer) if avg_ckpts: run_avg_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, global_step) train_sess.run(train_model.iterator.initializer, feed_dict={train_model.skip_count_placeholder: 0}) continue # Process step_result, accumulate stats, and write summary global_step, info["learning_rate"], step_summary = update_stats( stats, start_time, step_result) summary_writer.add_summary(step_summary, global_step) # Once in a while, we print statistics. if global_step - last_stats_step >= steps_per_stats: last_stats_step = global_step is_overflow = process_stats(stats, info, global_step, steps_per_stats, log_f) print_step_info(" ", global_step, info, get_best_results(hparams), log_f) if is_overflow: break # Reset statistics stats = init_stats() if global_step - last_eval_step >= steps_per_eval: last_eval_step = global_step utils.print_out("# Save eval, global step %d" % global_step) add_info_summaries(summary_writer, global_step, info) # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) # Evaluate on dev/test run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, sample_src_data, sample_tgt_data) run_internal_eval(eval_model, eval_sess, model_dir, hparams, summary_writer) if global_step - last_external_eval_step >= steps_per_external_eval: last_external_eval_step = global_step # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) run_sample_decode(infer_model, infer_sess, model_dir, hparams, summary_writer, sample_src_data, sample_tgt_data) run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer) if avg_ckpts: run_avg_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, global_step) # Done training loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "translate.ckpt"), global_step=global_step) (result_summary, _, final_eval_metrics) = (run_full_eval( model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams, summary_writer, sample_src_data, sample_tgt_data, avg_ckpts)) print_step_info("# Final, ", global_step, info, result_summary, log_f) utils.print_time("# Done training!", start_train_time) summary_writer.close() utils.print_out("# Start evaluating saved best models.") for metric in hparams.metrics: best_model_dir = getattr(hparams, "best_" + metric + "_dir") summary_writer = tf.summary.FileWriter( os.path.join(best_model_dir, summary_name), infer_model.graph) result_summary, best_global_step, _ = run_full_eval( best_model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams, summary_writer, sample_src_data, sample_tgt_data) print_step_info("# Best %s, " % metric, best_global_step, info, result_summary, log_f) summary_writer.close() if avg_ckpts: best_model_dir = getattr(hparams, "avg_best_" + metric + "_dir") summary_writer = tf.summary.FileWriter( os.path.join(best_model_dir, summary_name), infer_model.graph) result_summary, best_global_step, _ = run_full_eval( best_model_dir, infer_model, infer_sess, eval_model, eval_sess, hparams, summary_writer, sample_src_data, sample_tgt_data) print_step_info("# Averaged Best %s, " % metric, best_global_step, info, result_summary, log_f) summary_writer.close() return final_eval_metrics, global_step
def run_external_eval(infer_model, infer_sess, model_dir, hparams, summary_writer, save_best_dev=True, use_test_set=True, avg_ckpts=False, dev_infer_iterator_feed_dict=None, test_infer_iterator_feed_dict=None): """Compute external evaluation for both dev / test. Computes development and testing external evaluation (e.g. bleu, rouge) for given model. Args: infer_model: Inference model for which to compute perplexities. infer_sess: Inference TensorFlow session. model_dir: Directory from which to load inference model from. hparams: Model hyper-parameters. summary_writer: Summary writer for logging metrics to TensorBoard. use_test_set: Computes testing external evaluation if true; does not otherwise. Note that the development external evaluation is always computed regardless of value of this parameter. dev_infer_iterator_feed_dict: Feed dictionary for a TensorFlow session. Can be used to pass in additional inputs necessary for running the development external evaluation. test_infer_iterator_feed_dict: Feed dictionary for a TensorFlow session. Can be used to pass in additional inputs necessary for running the testing external evaluation. Returns: Triple containing development scores, testing scores and the TensorFlow Variable for the global step number, in this order. """ if dev_infer_iterator_feed_dict is None: dev_infer_iterator_feed_dict = {} if test_infer_iterator_feed_dict is None: test_infer_iterator_feed_dict = {} with infer_model.graph.as_default(): loaded_infer_model, global_step = model_helper.create_or_load_model( infer_model.model, model_dir, infer_sess, "infer") dev_src_file = "%s.%s" % (hparams.dev_prefix, hparams.src) dev_tgt_file = "%s.%s" % (hparams.dev_prefix, hparams.tgt) dev_infer_iterator_feed_dict[ infer_model.src_placeholder] = inference.load_data(dev_src_file) dev_infer_iterator_feed_dict[ infer_model.batch_size_placeholder] = hparams.infer_batch_size dev_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, dev_infer_iterator_feed_dict, dev_tgt_file, "dev", summary_writer, save_on_best=save_best_dev, avg_ckpts=avg_ckpts) test_scores = None if use_test_set and hparams.test_prefix: test_src_file = "%s.%s" % (hparams.test_prefix, hparams.src) test_tgt_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) test_infer_iterator_feed_dict[ infer_model.src_placeholder] = inference.load_data(test_src_file) test_infer_iterator_feed_dict[ infer_model.batch_size_placeholder] = hparams.infer_batch_size test_scores = _external_eval(loaded_infer_model, global_step, infer_sess, hparams, infer_model.iterator, test_infer_iterator_feed_dict, test_tgt_file, "test", summary_writer, save_on_best=False, avg_ckpts=avg_ckpts) return dev_scores, test_scores, global_step
def train(hparams, identity, scope=None, target_session=""): """main loop to train the dialogue model. identity is used.""" out_dir = hparams.out_dir steps_per_stats = hparams.steps_per_stats steps_per_internal_eval = 3 * steps_per_stats model_creator = diag_model.Model train_model = model_helper.create_train_model(model_creator, hparams, scope) model_dir = hparams.out_dir # Log and output files log_file = os.path.join(out_dir, identity+"log_%d" % time.time()) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) avg_step_time = 0.0 # load TensorFlow session and model config_proto = utils.get_config_proto( log_device_placement=hparams.log_device_placement, allow_soft_placement=True) train_sess = tf.Session( target=target_session, config=config_proto, graph=train_model.graph) train_handle = train_sess.run(train_model.train_iterator.string_handle()) with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( train_model.model, model_dir, train_sess, "train") # initialize summary writer summary_writer = tf.summary.FileWriter( os.path.join(out_dir, "train_log"), train_model.graph) last_stats_step = global_step last_eval_step = global_step # initialize training stats. step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0 checkpoint_total_count = 0.0 speed, train_ppl = 0.0, 0.0 start_train_time = time.time() utils.print_out( "# Start step %d, lr %g, %s" % (global_step, loaded_train_model.learning_rate.eval(session=train_sess), time.ctime()), log_f) # initialize iterators skip_count = hparams.batch_size * hparams.epoch_step utils.print_out("# Init train iterator, skipping %d elements" % skip_count) train_sess.run( train_model.train_iterator.initializer, feed_dict={train_model.skip_count_placeholder: skip_count}) # main training loop while global_step < hparams.num_train_steps: start_time = time.time() try: # run a step step_result = loaded_train_model.train(train_sess, train_handle) (_, step_loss, all_summaries, step_predict_count, step_summary, global_step, step_word_count, batch_size, _, _, words1, words2, mask1, mask2) = step_result hparams.epoch_step += 1 except tf.errors.OutOfRangeError: # finished an epoch hparams.epoch_step = 0 utils.print_out("# Finished an epoch, step %d." % global_step) train_sess.run( train_model.train_iterator.initializer, feed_dict={train_model.skip_count_placeholder: 0}) continue # Write step summary. summary_writer.add_summary(step_summary, global_step) for key in all_summaries: utils.add_summary(summary_writer, global_step, key, all_summaries[key]) # update statistics step_time += (time.time() - start_time) checkpoint_loss += (step_loss * batch_size) checkpoint_predict_count += step_predict_count checkpoint_total_count += float(step_word_count) if global_step - last_stats_step >= steps_per_stats: # print statistics for the previous epoch and save the model. last_stats_step = global_step avg_step_time = step_time / steps_per_stats utils.add_summary(summary_writer, global_step, "step_time", avg_step_time) train_ppl = utils.safe_exp(checkpoint_loss / checkpoint_predict_count) speed = checkpoint_total_count / (1000 * step_time) if math.isnan(train_ppl): break # Reset timer and loss. step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0 checkpoint_total_count = 0.0 # save the model loaded_train_model.saver.save( train_sess, os.path.join(out_dir, "dialogue.ckpt"), global_step=global_step) # print the dialogue if in debug mode if hparams.debug: utils.print_current_dialogue(words1, words2, mask1, mask2) # write out internal evaluation if global_step - last_eval_step >= steps_per_internal_eval: last_eval_step = global_step utils.print_out("# Internal Evaluation. global step %d" % global_step) utils.add_summary(summary_writer, global_step, "train_ppl", train_ppl) # finished training loaded_train_model.saver.save( train_sess, os.path.join(out_dir, "dialogue.ckpt"), global_step=global_step) result_summary = "" utils.print_out( "# Final, step %d lr %g " "step-time %.2f wps %.2fK ppl %.2f, %s, %s" % (global_step, loaded_train_model.learning_rate.eval(session=train_sess), avg_step_time, speed, train_ppl, result_summary, time.ctime()), log_f) utils.print_time("# Done training!", start_train_time) utils.print_out("# Start evaluating saved best models.") summary_writer.close()
def train(flags): """Train the policy gradient model. """ out_dir = flags.out_dir num_train_steps = flags.num_train_steps steps_per_infer = flags.steps_per_infer # Create model for train, infer mode model_creator = get_model_creator(flags) train_model = model_helper.create_train_model(flags, model_creator) infer_model = model_helper.create_infer_model(flags, model_creator) # TODO. set for distributed training and multi gpu config_proto = tf.ConfigProto(allow_soft_placement=True) config_proto.gpu_options.allow_growth = True # Session for train, infer train_sess = tf.Session( config=config_proto, graph=train_model.graph) infer_sess = tf.Session( config=config_proto, graph=infer_model.graph) # Load the train model if there's the file in the directory # otherwise, initialize vars in the train model with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( out_dir, train_model.model, train_sess) # Summary train_summary = "train_log" infer_summary = "infer_log" # Summary writer for train, infer train_summary_writer = tf.summary.FileWriter( os.path.join(out_dir, train_summary), train_model.graph) infer_summary_writer = tf.summary.FileWriter( os.path.join(out_dir, infer_summary)) # First evaluation run_infer(infer_model, out_dir, infer_sess) # Initialize step var last_infer_steps = global_step # Training loop while global_step < num_train_steps: output_tuple = loaded_train_model.train(train_sess) global_step = output_tuple.global_step train_summary = output_tuple.train_summary # Update train summary train_summary_writer.add_summary(train_summary, global_step) print('current global_step: {}'.format(global_step)) # Evaluate the model for steps_per_infer if global_step - last_infer_steps >= steps_per_infer: # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "rl.ckpt"), global_step) last_infer_steps = global_step output_tuple = run_infer(infer_model, out_dir, infer_sess) infer_summary = output_tuple.infer_summary # Update infer summary infer_summary_writer.add_summary(infer_summary, global_step) # Done training loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "rl.ckpt"), global_step) print('Train done')
def train(hps, scope=None, target_session=""): """Train a translation model.""" log_device_placement = hps.log_device_placement out_dir = hps.out_dir num_train_steps = hps.num_train_steps steps_per_stats = hps.steps_per_stats steps_per_external_eval = hps.steps_per_external_eval steps_per_eval = 100 * steps_per_stats if not steps_per_external_eval: steps_per_external_eval = 5 * steps_per_eval if hps.attention_architecture == "baseline": model_creator = AttentionModel else: model_creator = AttentionHistoryModel train_model = model_helper.create_train_model(model_creator, hps, scope) eval_model = model_helper.create_eval_model(model_creator, hps, scope) infer_model = model_helper.create_infer_model(model_creator, hps, scope) # Preload data for sample decoding. article_filenames = [] abstract_filenames = [] art_dir = hps.data_dir + '/article' abs_dir = hps.data_dir + '/abstract' for file in os.listdir(art_dir): if file.startswith(hps.dev_prefix): article_filenames.append(art_dir + "/" + file) for file in os.listdir(abs_dir): if file.startswith(hps.dev_prefix): abstract_filenames.append(abs_dir + "/" + file) # if random_decode: # """if this is a random sampling process during training""" decode_id = random.randint(0, len(article_filenames) - 1) single_article_file = article_filenames[decode_id] single_abstract_file = abstract_filenames[decode_id] dev_src_file = single_article_file dev_tgt_file = single_abstract_file sample_src_data = inference_base_model.load_data(dev_src_file) sample_tgt_data = inference_base_model.load_data(dev_tgt_file) summary_name = "train_log" model_dir = hps.out_dir # Log and output files log_file = os.path.join(out_dir, "log_%d" % time.time()) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) avg_step_time = 0.0 # TensorFlow model config_proto = utils.get_config_proto( log_device_placement=log_device_placement) train_sess = tf.Session(target=target_session, config=config_proto, graph=train_model.graph) eval_sess = tf.Session(target=target_session, config=config_proto, graph=eval_model.graph) infer_sess = tf.Session(target=target_session, config=config_proto, graph=infer_model.graph) with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( train_model.model, model_dir, train_sess, "train") # Summary writer summary_writer = tf.summary.FileWriter(os.path.join(out_dir, summary_name), train_model.graph) # First evaluation # run_full_eval( # model_dir, infer_model, infer_sess, # eval_model, eval_sess, hps, # summary_writer,sample_src_data,sample_tgt_data) last_stats_step = global_step last_eval_step = global_step last_external_eval_step = global_step # This is the training loop. stats = init_stats() speed, train_ppl = 0.0, 0.0 start_train_time = time.time() utils.print_out( "# Start step %d, lr %g, %s" % (global_step, loaded_train_model.learning_rate.eval( session=train_sess), time.ctime()), log_f) # Initialize all of the iterators skip_count = hps.batch_size * hps.epoch_step utils.print_out("# Init train iterator, skipping %d elements" % skip_count) train_sess.run(train_model.iterator.initializer, feed_dict={train_model.skip_count_placeholder: skip_count}) epoch_step = 0 while global_step < num_train_steps: ### Run a step ### start_time = time.time() try: step_result = loaded_train_model.train(train_sess) epoch_step += 1 except tf.errors.OutOfRangeError: # Finished going through the training dataset. Go to next epoch. epoch_step = 0 utils.print_out( "# Finished an epoch, step %d. Perform external evaluation" % global_step) run_sample_decode(infer_model, infer_sess, model_dir, hps, summary_writer, sample_src_data, sample_tgt_data) dev_scores, test_scores, _ = run_external_eval( infer_model, infer_sess, model_dir, hps, summary_writer) train_sess.run(train_model.iterator.initializer, feed_dict={train_model.skip_count_placeholder: 0}) continue # Write step summary and accumulate statistics global_step = update_stats(stats, summary_writer, start_time, step_result) # Once in a while, we print statistics. if global_step - last_stats_step >= steps_per_stats: last_stats_step = global_step is_overflow = check_stats(stats, global_step, steps_per_stats, hps, log_f) if is_overflow: break # Reset statistics stats = init_stats() if global_step - last_eval_step >= steps_per_eval: last_eval_step = global_step utils.print_out("# Save eval, global step %d" % global_step) utils.add_summary(summary_writer, global_step, "train_ppl", train_ppl) # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "summarized.ckpt"), global_step=global_step) # Evaluate on dev/test run_sample_decode(infer_model, infer_sess, model_dir, hps, summary_writer, sample_src_data, sample_tgt_data) dev_ppl, test_ppl = run_internal_eval(eval_model, eval_sess, model_dir, hps, summary_writer) if global_step - last_external_eval_step >= steps_per_external_eval: last_external_eval_step = global_step # Save checkpoint loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "summarized.ckpt"), global_step=global_step) run_sample_decode(infer_model, infer_sess, model_dir, hps, summary_writer, sample_src_data, sample_tgt_data) dev_scores, test_scores, _ = run_external_eval( infer_model, infer_sess, model_dir, hps, summary_writer) # Done training loaded_train_model.saver.save(train_sess, os.path.join(out_dir, "summarized.ckpt"), global_step=global_step) result_summary, _, dev_scores, test_scores, dev_ppl, test_ppl = run_full_eval( model_dir, infer_model, infer_sess, eval_model, eval_sess, hps, summary_writer, sample_src_data, sample_tgt_data) utils.print_out( "# Final, step %d lr %g " "step-time %.2f wps %.2fK ppl %.2f, %s, %s" % (global_step, loaded_train_model.learning_rate.eval(session=train_sess), avg_step_time, speed, train_ppl, result_summary, time.ctime()), log_f) utils.print_time("# Done training!", start_train_time) summary_writer.close() utils.print_out("# Start evaluating saved best models.") for metric in hps.metrics: best_model_dir = getattr(hps, "best_" + metric + "_dir") summary_writer = tf.summary.FileWriter( os.path.join(best_model_dir, summary_name), infer_model.graph) result_summary, best_global_step, _, _, _, _ = run_full_eval( best_model_dir, infer_model, infer_sess, eval_model, eval_sess, hps, summary_writer, sample_src_data, sample_tgt_data) utils.print_out( "# Best %s, step %d " "step-time %.2f wps %.2fK, %s, %s" % (metric, best_global_step, avg_step_time, speed, result_summary, time.ctime()), log_f) summary_writer.close() return (dev_scores, test_scores, dev_ppl, test_ppl, global_step)
def train(hparams): """Train a seq2seq model.""" log_device_placement = hparams.log_device_placement out_dir = hparams.out_dir num_train_steps = hparams.num_train_steps steps_per_stats = hparams.steps_per_stats model_creator = model.Model train_model = model_helper.create_train_model(model_creator, hparams) summary_name = "train_log" model_dir = hparams.out_dir # Log and output files log_file = os.path.join(out_dir, "log_%d" % time.time()) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_files=%s" % log_file, log_f) # TensorFlow model config_proto = utils.get_config_proto( log_device_placement=log_device_placement, num_intra_threads=hparams.num_intra_threads, num_inter_threads=hparams.num_inter_threads) train_sess = tf.Session(config=config_proto, graph=train_model.graph) with train_model.graph.as_default(): loaded_train_model, global_step = model_helper.create_or_load_model( train_model.model, model_dir, train_sess, "train") # Summary writer summary_writer = tf.summary.FileWriter( os.path.join(out_dir, summary_name), train_model.graph) last_stats_step = global_step # This is the training loop. stats, info, start_train_time = before_train( loaded_train_model, train_model, train_sess, global_step, hparams, log_f) while global_step < num_train_steps: ### Run a step ### start_time = time.time() try: step_result = loaded_train_model.train(train_sess) hparams.epoch_step += 1 except tf.errors.OutOfRangeError: # Finished going through the training dataset. Go to next epoch. hparams.epoch_step = 0 utils.print_out("# Finished an epoch, step %d." % global_step) train_sess.run( train_model.iterator.initializer, feed_dict={train_model.skip_count_placeholder: 0}) continue # Process step_result, accumulate stats, and write summary global_step, info["learning_rate"], step_summary = update_stats( stats, start_time, step_result) summary_writer.add_summary(step_summary, global_step) # Once in a while, we print statistics. if global_step - last_stats_step >= steps_per_stats: last_stats_step = global_step is_overflow = process_stats(stats, info, global_step, steps_per_stats, log_f) print_step_info(" ", global_step, info, log_f) if is_overflow: break # Reset statistics stats = init_stats() # Done training loaded_train_model.saver.save( train_sess, os.path.join(out_dir, "seq2seq.ckpt"), global_step=global_step) summary_writer.close() return global_step
def self_play_eval_fn(hparams, identity, num_workers=1, jobid=0, scope=None, target_session=''): """This is the single worker self play. Mostly used for self play evaluation. identity is used here to distinguish between workers. """ model_creator = diag_model.Model mutable_model = model_helper.create_selfplay_model( model_creator, True, # mutable is True num_workers, jobid, hparams=hparams, scope=scope) immutable_model = model_helper.create_selfplay_model( model_creator, False, # mutable is False num_workers, jobid, hparams=hparams, scope=scope) mutable_sess = tf.Session(graph=mutable_model.graph, config=tf.ConfigProto( allow_soft_placement=True, device_count={'GPU': hparams.num_gpus})) immutable_sess = tf.Session(graph=immutable_model.graph, config=tf.ConfigProto( allow_soft_placement=True, device_count={'GPU': hparams.num_gpus})) # number of steps per external eval steps_per_external_eval = 10 # force conducting a self play at the beginning last_external_eval_step = -1 * steps_per_external_eval print('hparams.self_play_pretrain_dir=', hparams.self_play_pretrain_dir) print('steps_per_external_eval=', steps_per_external_eval) writer_path = os.path.join(hparams.out_dir, identity + hparams.task_type + '_log') summary_writer = tf.summary.FileWriter(writer_path, mutable_sess.graph) print('summary_writer estabilished at', writer_path) # waiting for checkpoints and loop forever latest_ckpt = None while True: latest_ckpt = tf.contrib.training.wait_for_new_checkpoint( hparams.out_dir, latest_ckpt) print('got checkpoint', latest_ckpt) # get the global_step variable first with mutable_model.graph.as_default(): # first initialize to avoid encountering missing component for adam optimizer _, global_step = model_helper.create_or_load_model( mutable_model.model, hparams.out_dir, mutable_sess, hparams.task_type) # valid evaluation step if (not hparams.eval_forever) or (global_step - last_external_eval_step >= steps_per_external_eval): # if eval_forever is disabled, we will do one selfplay evalation # otherwise, we will wait until certain number of timesteps are elapsed. last_external_eval_step = global_step print('do single worker evaluation') single_worker_selfplay(mutable_model, immutable_model, mutable_sess, immutable_sess, hparams.self_play_eval_data, hparams.self_play_eval_kb, global_step, hparams, summary_writer) else: print('Wait until steps_per_external_eval is reached.', global_step, last_external_eval_step, steps_per_external_eval) if not hparams.eval_forever: break # if eval_foever is disabled, we only evaluate once mutable_sess.close() immutable_sess.close()