def eval_bleu(self): self.logger.info('Evaluate dev BLEU') start = time.time() self.model.eval() avg_bleus = [] dump_dir = self.args.dump_dir with torch.no_grad(): for pair in self.pairs: self.logger.info('--> {}'.format(pair)) src_lang, tgt_lang = pair.split('2') src_lang_idx = self.data_manager.lang_vocab[src_lang] tgt_lang_idx = self.data_manager.lang_vocab[tgt_lang] logit_mask = self.data_manager.logit_masks[tgt_lang] data = self.data_manager.translate_data[pair] src_batches = data['src_batches'] sorted_idxs = data['sorted_idxs'] ref_file = data['ref_file'] all_best_trans, all_beam_trans = self._translate(src_batches, sorted_idxs, src_lang_idx, tgt_lang_idx, logit_mask) all_best_trans = ''.join(all_best_trans) best_trans_file = join(dump_dir, '{}_val_trans.txt.bpe'.format(pair)) open(best_trans_file, 'w').close() with open(best_trans_file, 'w') as fout: fout.write(all_best_trans) all_beam_trans = ''.join(all_beam_trans) beam_trans_file = join(dump_dir, '{}_beam_trans.txt.bpe'.format(pair)) open(beam_trans_file, 'w').close() with open(beam_trans_file, 'w') as fout: fout.write(all_beam_trans) # merge BPE nobpe_best_trans_file = join(dump_dir, '{}_val_trans.txt'.format(pair)) ut.remove_bpe(best_trans_file, nobpe_best_trans_file) nobpe_beam_trans_file = join(dump_dir, '{}_beam_trans.txt'.format(pair)) ut.remove_bpe(beam_trans_file, nobpe_beam_trans_file) # calculate BLEU bleu, msg = ut.calc_bleu(self.args.bleu_script, nobpe_best_trans_file, ref_file) self.logger.info(msg) avg_bleus.append(bleu) self.stats[pair]['dev_bleus'].append(bleu) # save translation with BLEU score for future reference trans_file = '{}-{}'.format(nobpe_best_trans_file, bleu) shutil.copyfile(nobpe_best_trans_file, trans_file) beam_file = '{}-{}'.format(nobpe_beam_trans_file, bleu) shutil.copyfile(nobpe_beam_trans_file, beam_file) avg_bleu = sum(avg_bleus) / len(avg_bleus) self.stats['avg_bleus'].append(avg_bleu) self.logger.info('avg_bleu = {}'.format(avg_bleu)) self.logger.info('Done evaluating dev BLEU, it takes {} seconds'.format(ut.format_seconds(time.time() - start)))
def train(src, tgt, lengths, model, optimizer, criterion, is_train=True, teacher_forcing_ratio=0.8): """one step minibatch training Args: src (tensor): source data tgt (tensor): target data model (child class of nn.Module): seq2seq model optimizer (torch.optim) criterion: loss function is_train (bool): if True, parameters are upgraded by backpropagain. Default: True teacher_forcing_ratio (float): the probability of inputting ground truth Default:[0, 1] Returns: loss (float): averaged loss of all tokens """ y_pred = model(src, tgt, lengths, teacher_forcing_ratio) loss = criterion(y_pred, tgt) bleu = 0 if is_train: optimizer.zero_grad() loss.backward() optimizer.step() else: tgt = tgt.tolist() _, y_pred = y_pred.max(1) # shape = (batch_size, seq_len) y_pred = y_pred.tolist() bleu = utils.calc_bleu(y_pred, tgt) return loss.item(), bleu
_loss = sess.run(loss) logging.info("# Test evaluation") _, _eval_summary = sess.run([eval_init_op, eval_summaries]) summary_writer.add_summary(_eval_summary, epoch) logging.info("# Get hypotheses") hypotheses = get_hypotheses(num_eval_batches, num_eval_samples, sess, y_hat, m.idx2token) logging.info("# Write results") model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss) if not os.path.exsits(hp.evaldir): os.makedirs(hp.evaldir) translation = os.path.join(hp.evaldir, model_output) with open(translation, "w") as f: f.write("\n".join(hypotheses)) logging.info("# Calc bleu score and append it to translation") calc_bleu(hp.eval3, translation) logging.info("# Save model") ckpt_name = os.ptah.join(hp.logdir, model_output) saver.save(sess, ckpt_name, global_step=_gs) logging.info("After training of {} epochs, {} has been saved.".format(epoch, ckpt_name)) logging.info("# Fall back to train mode") sess.run(train_init_op) summary_writer.close() logging.info("Done")
logging.info("# get hypotheses") hypotheses, refs_result = get_hypotheses(num_test_batches, num_test_samples, sess, y_hat, refs, m.idx2token) # 将原始的结果写到本地 logging.info("write references") result_output = "refs" if not os.path.exists(hp.test_result): os.makedirs(hp.test_result) ref_path = os.path.join(hp.test_result, result_output) with open(ref_path, 'w', encoding='utf-8') as fout: _refs = [] for r in refs_result: words = r.decode('utf-8').split() s = [word.replace("▁", " ") for word in words] # remove bpe symbols sent = ''.join(s) _refs.append(sent.strip()) fout.write("\n".join(_refs)) logging.info("# write results") result_output = "trans" if not os.path.exists(hp.test_result): os.makedirs(hp.test_result) translation = os.path.join(hp.test_result, result_output) with open(translation, 'w', encoding='utf-8') as fout: fout.write("\n".join(hypotheses)) logging.info("# calc bleu score and write it to disk") calc_bleu(ref_path, translation, 'test')
words = r.decode('utf-8').split() s = [word.replace("▁", " ") for word in words] # remove bpe symbols sent = ''.join(s) _refs.append(sent.strip()) fout.write("\n".join(_refs)) logging.info("# write results") model_output = "trans_ The%02dL Epoch loss is %.2f" % (epoch, _loss) if not os.path.exists(hp.eval_result): os.makedirs(hp.eval_result) translation_path = os.path.join(hp.eval_result, model_output) with open(translation_path, 'w', encoding='utf-8') as fout: fout.write("\n".join(hypotheses)) logging.info("# calc bleu score and write it to disk") calc_bleu(ref_path, translation_path, 'eval', epoch) logging.info("# save models") model_output = "The%02dL Epoch loss is %.2f" % (epoch, _loss) ckpt_name = os.path.join(hp.modeldir, model_output) saver.save(sess, ckpt_name, global_step=_gs) logging.info( "after training of {} epochs, {} has been saved.".format( epoch, ckpt_name)) logging.info("# fall back to train mode") sess.run(train_init_op) summary_writer.close() logging.info("Done")
def train_(self, epochs): train_batches, num_train_batches, num_train_samples = get_batch( '../data/iwslt2016/segmented/train.de.bpe', '../data/iwslt2016/segmented/train.en.bpe', self.sequence_length, self.sequence_length, self.vocab_file, self.batch_size, shuffle=True) eval_batches, num_eval_batches, num_eval_samples = get_batch( '../data/iwslt2016/segmented/eval.de.bpe', '../data/iwslt2016/segmented/eval.en.bpe', 100000, 100000, self.vocab_file, self.batch_size, shuffle=False) iter = tf.data.Iterator.from_structure(train_batches.output_types, train_batches.output_shapes) xs, ys = iter.get_next() train_init_op = iter.make_initializer(train_batches) eval_init_op = iter.make_initializer(eval_batches) loss, train_op, global_step, train_summaries = self.model.train(xs, ys) y_hat, eval_summaries = self.model.eval(xs, ys) logging.info("# Session") with tf.Session() as sess: ckpt = tf.train.latest_checkpoint(self.model_dir) if ckpt is None: logging.info("Initializing from scratch") sess.run(tf.global_variables_initializer()) save_variable_specs(os.path.join('../data/log/1', "specs")) else: self.saver.restore(sess, ckpt) summary_writer = tf.summary.FileWriter(self.model_dir, sess.graph) sess.run(train_init_op) total_steps = epochs * num_train_batches _gs = sess.run(global_step) for i in tqdm(range(_gs, total_steps + 1)): _, _gs, _summary = sess.run( [train_op, global_step, train_summaries]) epoch = math.ceil(_gs / num_train_batches) summary_writer.add_summary(_summary, _gs) if _gs and _gs % num_train_batches == 0: logging.info("epoch {} is done".format(epoch)) _loss = sess.run(loss) # train loss logging.info("# test evaluation") _, _eval_summaries = sess.run( [eval_init_op, eval_summaries]) summary_writer.add_summary(_eval_summaries, _gs) logging.info("# get hypotheses") hypotheses = get_hypotheses(num_eval_batches, num_eval_samples, sess, y_hat, self.model.index_char) logging.info("# write results") model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss) if not os.path.exists('data/eval/1'): os.makedirs('../data/eval/1') translation = os.path.join('../data/eval/1', model_output) with open(translation, 'w') as fout: fout.write("\n".join(hypotheses)) logging.info( "# calc bleu score and append it to translation") calc_bleu('../data/iwslt2016/prepro/eval.en', translation) logging.info("# save models") self.saver.save(sess, os.path.join(self.model_dir, 'transformer.dat'), global_step=_gs) sess.run(train_init_op) summary_writer.close() logging.info("Done")
summary_writer.add_summary(_eval_summaries, _gs) logging.info("# get hypotheses") hypotheses = get_hypotheses(num_eval_batches, num_eval_samples, sess, y_hat, m.idx2token) logging.info("# write results") model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss) if not os.path.exists(hp.evaldir): os.makedirs(hp.evaldir) translation = os.path.join(hp.evaldir, model_output) with open(translation, 'w', encoding="utf-8") as fout: fout.write("\n".join(hypotheses)) logging.info("# calc bleu score and append it to translation") try: calc_bleu(os.path.join(hp.data_dir, hp.eval3), translation) except: logging.info("error in calc bleu") logging.info("# save models") ckpt_name = os.path.join(hp.logdir, model_output) saver.save(sess, ckpt_name, global_step=_gs) logging.info( "after training of {} epochs, {} has been saved.".format( epoch, ckpt_name)) logging.info("# fall back to train mode") sess.run(train_init_op) summary_writer.close() logging.info("Done")
test_init_op = iter.make_initializer(test_batches) logging.info("# Load model") m = Transformer(hp) y_hat, _ = m.eval(xs, ys) logging.info("# Session") with tf.Session() as sess: ckpt_ = tf.train.latest_checkpoint(hp.ckpt) ckpt = hp.ckpt if ckpt_ is None else ckpt_ # None: ckpt is a file. otherwise dir. saver = tf.train.Saver() saver.restore(sess, ckpt) sess.run(test_init_op) logging.info("# get hypotheses") hypotheses = get_hypotheses(num_test_batches, num_test_samples, sess, y_hat, m.idx2token) logging.info("# write results") model_output = ckpt.split("/")[-1] if not os.path.exists(hp.testdir): os.makedirs(hp.testdir) translation = os.path.join(hp.testdir, model_output) with open(translation, 'w') as fout: fout.write("\n".join(hypotheses)) logging.info("# calc bleu score and append it to translation") calc_bleu(hp.test2, translation)
import os import sys from utils import calc_bleu ref = sys.argv[1] transdir = sys.argv[2] for trans in os.listdir(transdir): if "B" not in trans: transpath = os.path.join(transdir, trans) print(transpath) calc_bleu(ref, transpath)
logging.info("# Session") with tf.Session() as sess: ckpt_ = tf.train.latest_checkpoint(hp.ckpt) ckpt = hp.ckpt if ckpt_ is None else ckpt_ # None: ckpt is a file. otherwise dir. saver = tf.train.Saver() saver.restore(sess, ckpt) if hp.write_weights: weightsdir=hp.weightsdir if not os.path.exists(weightsdir): os.makedirs(weightsdir) for v in tf.trainable_variables(): weights=sess.run(v) layername=v.name.replace("/","_") np.savetxt(os.path.join(weightsdir,layername+".txt"),weights) sess.run(test_init_op) logging.info("# get hypotheses") hypotheses = get_hypotheses(num_test_batches, num_test_samples, sess, y_hat, m.idx2token) logging.info("# write results") model_output = ckpt.split("/")[-1] if not os.path.exists(hp.testdir): os.makedirs(hp.testdir) translation = os.path.join(hp.testdir, model_output) with open(translation, 'w',encoding="utf-8") as fout: fout.write("\n".join(hypotheses)) logging.info("# calc bleu score and append it to translation") calc_bleu(test2path, translation)