def eval_bleu(self):
self.logger.info('Evaluate dev BLEU')
start = time.time()
self.model.eval()
avg_bleus = []
dump_dir = self.args.dump_dir
with torch.no_grad():
for pair in self.pairs:
self.logger.info('--> {}'.format(pair))
src_lang, tgt_lang = pair.split('2')
src_lang_idx = self.data_manager.lang_vocab[src_lang]
tgt_lang_idx = self.data_manager.lang_vocab[tgt_lang]
logit_mask = self.data_manager.logit_masks[tgt_lang]
data = self.data_manager.translate_data[pair]
src_batches = data['src_batches']
sorted_idxs = data['sorted_idxs']
ref_file = data['ref_file']
all_best_trans, all_beam_trans = self._translate(src_batches, sorted_idxs, src_lang_idx, tgt_lang_idx, logit_mask)
all_best_trans = ''.join(all_best_trans)
best_trans_file = join(dump_dir, '{}_val_trans.txt.bpe'.format(pair))
open(best_trans_file, 'w').close()
with open(best_trans_file, 'w') as fout:
fout.write(all_best_trans)
all_beam_trans = ''.join(all_beam_trans)
beam_trans_file = join(dump_dir, '{}_beam_trans.txt.bpe'.format(pair))
open(beam_trans_file, 'w').close()
with open(beam_trans_file, 'w') as fout:
fout.write(all_beam_trans)
# merge BPE
nobpe_best_trans_file = join(dump_dir, '{}_val_trans.txt'.format(pair))
ut.remove_bpe(best_trans_file, nobpe_best_trans_file)
nobpe_beam_trans_file = join(dump_dir, '{}_beam_trans.txt'.format(pair))
ut.remove_bpe(beam_trans_file, nobpe_beam_trans_file)
# calculate BLEU
bleu, msg = ut.calc_bleu(self.args.bleu_script, nobpe_best_trans_file, ref_file)
self.logger.info(msg)
avg_bleus.append(bleu)
self.stats[pair]['dev_bleus'].append(bleu)
# save translation with BLEU score for future reference
trans_file = '{}-{}'.format(nobpe_best_trans_file, bleu)
shutil.copyfile(nobpe_best_trans_file, trans_file)
beam_file = '{}-{}'.format(nobpe_beam_trans_file, bleu)
shutil.copyfile(nobpe_beam_trans_file, beam_file)
avg_bleu = sum(avg_bleus) / len(avg_bleus)
self.stats['avg_bleus'].append(avg_bleu)
self.logger.info('avg_bleu = {}'.format(avg_bleu))
self.logger.info('Done evaluating dev BLEU, it takes {} seconds'.format(ut.format_seconds(time.time() - start)))
def train(src,
tgt,
lengths,
model,
optimizer,
criterion,
is_train=True,
teacher_forcing_ratio=0.8):
"""one step minibatch training
Args:
src (tensor): source data
tgt (tensor): target data
model (child class of nn.Module): seq2seq model
optimizer (torch.optim)
criterion: loss function
is_train (bool): if True, parameters are upgraded by backpropagain. Default: True
teacher_forcing_ratio (float): the probability of inputting ground truth Default:[0, 1]
Returns:
loss (float): averaged loss of all tokens
"""
y_pred = model(src, tgt, lengths, teacher_forcing_ratio)
loss = criterion(y_pred, tgt)
bleu = 0
if is_train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
tgt = tgt.tolist()
_, y_pred = y_pred.max(1) # shape = (batch_size, seq_len)
y_pred = y_pred.tolist()
bleu = utils.calc_bleu(y_pred, tgt)
return loss.item(), bleu
_loss = sess.run(loss)
logging.info("# Test evaluation")
_, _eval_summary = sess.run([eval_init_op, eval_summaries])
summary_writer.add_summary(_eval_summary, epoch)
logging.info("# Get hypotheses")
hypotheses = get_hypotheses(num_eval_batches, num_eval_samples, sess, y_hat, m.idx2token)
logging.info("# Write results")
model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss)
if not os.path.exsits(hp.evaldir): os.makedirs(hp.evaldir)
translation = os.path.join(hp.evaldir, model_output)
with open(translation, "w") as f:
f.write("\n".join(hypotheses))
logging.info("# Calc bleu score and append it to translation")
calc_bleu(hp.eval3, translation)
logging.info("# Save model")
ckpt_name = os.ptah.join(hp.logdir, model_output)
saver.save(sess, ckpt_name, global_step=_gs)
logging.info("After training of {} epochs, {} has been saved.".format(epoch, ckpt_name))
logging.info("# Fall back to train mode")
sess.run(train_init_op)
summary_writer.close()
logging.info("Done")
logging.info("# get hypotheses")
hypotheses, refs_result = get_hypotheses(num_test_batches,
num_test_samples, sess, y_hat,
refs, m.idx2token)
# 将原始的结果写到本地
logging.info("write references")
result_output = "refs"
if not os.path.exists(hp.test_result): os.makedirs(hp.test_result)
ref_path = os.path.join(hp.test_result, result_output)
with open(ref_path, 'w', encoding='utf-8') as fout:
_refs = []
for r in refs_result:
words = r.decode('utf-8').split()
s = [word.replace("▁", " ")
for word in words] # remove bpe symbols
sent = ''.join(s)
_refs.append(sent.strip())
fout.write("\n".join(_refs))
logging.info("# write results")
result_output = "trans"
if not os.path.exists(hp.test_result): os.makedirs(hp.test_result)
translation = os.path.join(hp.test_result, result_output)
with open(translation, 'w', encoding='utf-8') as fout:
fout.write("\n".join(hypotheses))
logging.info("# calc bleu score and write it to disk")
calc_bleu(ref_path, translation, 'test')
words = r.decode('utf-8').split()
s = [word.replace("▁", " ")
for word in words] # remove bpe symbols
sent = ''.join(s)
_refs.append(sent.strip())
fout.write("\n".join(_refs))
logging.info("# write results")
model_output = "trans_ The%02dL Epoch loss is %.2f" % (epoch,
_loss)
if not os.path.exists(hp.eval_result): os.makedirs(hp.eval_result)
translation_path = os.path.join(hp.eval_result, model_output)
with open(translation_path, 'w', encoding='utf-8') as fout:
fout.write("\n".join(hypotheses))
logging.info("# calc bleu score and write it to disk")
calc_bleu(ref_path, translation_path, 'eval', epoch)
logging.info("# save models")
model_output = "The%02dL Epoch loss is %.2f" % (epoch, _loss)
ckpt_name = os.path.join(hp.modeldir, model_output)
saver.save(sess, ckpt_name, global_step=_gs)
logging.info(
"after training of {} epochs, {} has been saved.".format(
epoch, ckpt_name))
logging.info("# fall back to train mode")
sess.run(train_init_op)
summary_writer.close()
logging.info("Done")
def train_(self, epochs):
train_batches, num_train_batches, num_train_samples = get_batch(
'../data/iwslt2016/segmented/train.de.bpe',
'../data/iwslt2016/segmented/train.en.bpe',
self.sequence_length,
self.sequence_length,
self.vocab_file,
self.batch_size,
shuffle=True)
eval_batches, num_eval_batches, num_eval_samples = get_batch(
'../data/iwslt2016/segmented/eval.de.bpe',
'../data/iwslt2016/segmented/eval.en.bpe',
100000,
100000,
self.vocab_file,
self.batch_size,
shuffle=False)
iter = tf.data.Iterator.from_structure(train_batches.output_types,
train_batches.output_shapes)
xs, ys = iter.get_next()
train_init_op = iter.make_initializer(train_batches)
eval_init_op = iter.make_initializer(eval_batches)
loss, train_op, global_step, train_summaries = self.model.train(xs, ys)
y_hat, eval_summaries = self.model.eval(xs, ys)
logging.info("# Session")
with tf.Session() as sess:
ckpt = tf.train.latest_checkpoint(self.model_dir)
if ckpt is None:
logging.info("Initializing from scratch")
sess.run(tf.global_variables_initializer())
save_variable_specs(os.path.join('../data/log/1', "specs"))
else:
self.saver.restore(sess, ckpt)
summary_writer = tf.summary.FileWriter(self.model_dir, sess.graph)
sess.run(train_init_op)
total_steps = epochs * num_train_batches
_gs = sess.run(global_step)
for i in tqdm(range(_gs, total_steps + 1)):
_, _gs, _summary = sess.run(
[train_op, global_step, train_summaries])
epoch = math.ceil(_gs / num_train_batches)
summary_writer.add_summary(_summary, _gs)
if _gs and _gs % num_train_batches == 0:
logging.info("epoch {} is done".format(epoch))
_loss = sess.run(loss) # train loss
logging.info("# test evaluation")
_, _eval_summaries = sess.run(
[eval_init_op, eval_summaries])
summary_writer.add_summary(_eval_summaries, _gs)
logging.info("# get hypotheses")
hypotheses = get_hypotheses(num_eval_batches,
num_eval_samples, sess, y_hat,
self.model.index_char)
logging.info("# write results")
model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss)
if not os.path.exists('data/eval/1'):
os.makedirs('../data/eval/1')
translation = os.path.join('../data/eval/1', model_output)
with open(translation, 'w') as fout:
fout.write("\n".join(hypotheses))
logging.info(
"# calc bleu score and append it to translation")
calc_bleu('../data/iwslt2016/prepro/eval.en', translation)
logging.info("# save models")
self.saver.save(sess,
os.path.join(self.model_dir,
'transformer.dat'),
global_step=_gs)
sess.run(train_init_op)
summary_writer.close()
logging.info("Done")
summary_writer.add_summary(_eval_summaries, _gs)
logging.info("# get hypotheses")
hypotheses = get_hypotheses(num_eval_batches, num_eval_samples,
sess, y_hat, m.idx2token)
logging.info("# write results")
model_output = "iwslt2016_E%02dL%.2f" % (epoch, _loss)
if not os.path.exists(hp.evaldir): os.makedirs(hp.evaldir)
translation = os.path.join(hp.evaldir, model_output)
with open(translation, 'w', encoding="utf-8") as fout:
fout.write("\n".join(hypotheses))
logging.info("# calc bleu score and append it to translation")
try:
calc_bleu(os.path.join(hp.data_dir, hp.eval3), translation)
except:
logging.info("error in calc bleu")
logging.info("# save models")
ckpt_name = os.path.join(hp.logdir, model_output)
saver.save(sess, ckpt_name, global_step=_gs)
logging.info(
"after training of {} epochs, {} has been saved.".format(
epoch, ckpt_name))
logging.info("# fall back to train mode")
sess.run(train_init_op)
summary_writer.close()
logging.info("Done")
test_init_op = iter.make_initializer(test_batches)
logging.info("# Load model")
m = Transformer(hp)
y_hat, _ = m.eval(xs, ys)
logging.info("# Session")
with tf.Session() as sess:
ckpt_ = tf.train.latest_checkpoint(hp.ckpt)
ckpt = hp.ckpt if ckpt_ is None else ckpt_ # None: ckpt is a file. otherwise dir.
saver = tf.train.Saver()
saver.restore(sess, ckpt)
sess.run(test_init_op)
logging.info("# get hypotheses")
hypotheses = get_hypotheses(num_test_batches, num_test_samples, sess, y_hat, m.idx2token)
logging.info("# write results")
model_output = ckpt.split("/")[-1]
if not os.path.exists(hp.testdir): os.makedirs(hp.testdir)
translation = os.path.join(hp.testdir, model_output)
with open(translation, 'w') as fout:
fout.write("\n".join(hypotheses))
logging.info("# calc bleu score and append it to translation")
calc_bleu(hp.test2, translation)
import os
import sys
from utils import calc_bleu
ref = sys.argv[1]
transdir = sys.argv[2]
for trans in os.listdir(transdir):
if "B" not in trans:
transpath = os.path.join(transdir, trans)
print(transpath)
calc_bleu(ref, transpath)
logging.info("# Session")
with tf.Session() as sess:
ckpt_ = tf.train.latest_checkpoint(hp.ckpt)
ckpt = hp.ckpt if ckpt_ is None else ckpt_ # None: ckpt is a file. otherwise dir.
saver = tf.train.Saver()
saver.restore(sess, ckpt)
if hp.write_weights:
weightsdir=hp.weightsdir
if not os.path.exists(weightsdir):
os.makedirs(weightsdir)
for v in tf.trainable_variables():
weights=sess.run(v)
layername=v.name.replace("/","_")
np.savetxt(os.path.join(weightsdir,layername+".txt"),weights)
sess.run(test_init_op)
logging.info("# get hypotheses")
hypotheses = get_hypotheses(num_test_batches, num_test_samples, sess, y_hat, m.idx2token)
logging.info("# write results")
model_output = ckpt.split("/")[-1]
if not os.path.exists(hp.testdir): os.makedirs(hp.testdir)
translation = os.path.join(hp.testdir, model_output)
with open(translation, 'w',encoding="utf-8") as fout:
fout.write("\n".join(hypotheses))
logging.info("# calc bleu score and append it to translation")
calc_bleu(test2path, translation)
