def end_evaluation(self, dev_data, trg_vocab):
if len(self.predictions) != 0:
self.state.bleu_scores.append(calculate_bleu(self.predictions,
trg_dev,
trg_vocab))
# Adding previous dev perplexities if available
if len(self.state.dev_perplexities) > 0:
prev_dev_ppl = self.state.dev_ppl()
else:
prev_dev_ppl = None
# Update the dev perplexities
self.state.dev_perplexities.append(math.exp(self.dev_loss / self.dev_loss_ctr))
# Generate one line report
dev_ppl = self.state.dev_ppl()
if prev_dev_ppl is None:
dev_ppl_report = "DEV_PPL=%.3f" % dev_ppl
else:
dev_ppl_report = "DEV_PPL=(%.3f -> %.3f)" % (prev_dev_ppl, dev_ppl)
# Appending BLEU scores information if generated
if len(self.state.bleu_scores) != 0:
dev_ppl_report += ", BLEU=%s" % self.state.bleu()
# Reporting
log.info("End Evaluation: %s" % (dev_ppl_report))
def begin_evaluation(self):
log.info("Decoding Started")
self.time = time.time()
self.predictions = []
self.test_loss = 0
self.test_loss_ctr = 0
self.attentions = []
def end_epoch(self, new_data_arrangement):
self.state.finished_epoch += 1
self.state.batch_indexes = new_data_arrangement
self.state.time_spent.append(time.time() - self.time)
self.state.perplexities.append(math.exp(self.epoch_loss / self.epoch_update_counter))
log.info("Epoch %d finished! ppl=%.3f, time=%.4f mins" % (self.state.finished_epoch,
self.state.ppl(),
self.state.last_time() / 60))
def batch_update(self, id, loss=0, batch_size=1, col_size=1):
ppl = math.exp(float(loss))
self.epoch_loss += loss
self.epoch_update_counter += 1
self.state.trained_sentence += batch_size
log.info("[%d] Sentence trained: %d, Batch(PPL=%.3f, size=(%d,%d), wps=%d, id=%d)" % (self.state.finished_epoch + 1,
self.state.trained_sentence,
ppl,
batch_size, col_size,
abs((batch_size * col_size) / (time.time() - self.batch_time)),
id))
def begin_epoch(self):
# Number of training sentences
self.trained = 0
# To measure ppl
self.epoch_loss = 0
self.epoch_update_counter = 0
# To measure time
self.time = time.time()
# To measure number of trained sentence
self.state.trained_sentence = 0
# Verbose
log.info("Start Epoch %d" % (self.state.finished_epoch + 1))
def end_evaluation(self, test_data, trg_vocab):
log.info("Decoding Finished, starting evaluation if reference is provided.")
self.state.time_spent.append(time.time() - self.time)
ref = test_data.trg_path
if ref is not None:
self.state.bleu_scores.append(calculate_bleu(self.predictions, ref, trg_vocab))
self.state.perplexities.append(math.exp(self.test_loss / self.test_loss_ctr))
# Creating evaluation string
eval_string = "Time=%.2f mins" % (self.state.last_time() / 60)
if ref is not None:
eval_string += " " + ("BLEU=%s, test_ppl=%.3f" % (str(self.state.bleu()), self.state.ppl()))
log.info("Evaluation Finished!", eval_string)
def save(model, out_file):
if not out_file.endswith(".zip"):
out_file = out_file + ".zip"
tmpdir = tempfile.mkdtemp()
log.info("Saving model to", out_file)
# Saving Specification
pickle_save(os.path.join(tmpdir, SPEC), model.specification.__dict__)
# Saving vocabularies
pickle_save(os.path.join(tmpdir, SRC_VOC), model.src_vocab)
pickle_save(os.path.join(tmpdir, TRG_VOC), model.trg_vocab)
# Saving optimizer state
chainer.serializers.save_npz(os.path.join(tmpdir, OPTIMIZER),
model.optimizer)
# Saving training state
pickle_save(os.path.join(tmpdir, STATE), model.training_state)
# Saving Weight
chainer.serializers.save_npz(os.path.join(tmpdir, WEIGHT),
model.chainer_model)
# Saving Lexicon
if model.lexicon is not None:
pickle_save(os.path.join(tmpdir, LEXICON), model.lexicon)
# Saving BPE codec
if hasattr(model, "bpe_codec"):
pickle_save(os.path.join(tmpdir, BPE_CODEC), model.bpe_codec)
# Zipping
zf = zipfile.ZipFile(out_file, mode="w", compression=zipfile.ZIP_DEFLATED)
try:
write_zip(zf, os.path.join(tmpdir, SRC_VOC))
write_zip(zf, os.path.join(tmpdir, TRG_VOC))
write_zip(zf, os.path.join(tmpdir, STATE))
write_zip(zf, os.path.join(tmpdir, WEIGHT))
write_zip(zf, os.path.join(tmpdir, SPEC))
write_zip(zf, os.path.join(tmpdir, OPTIMIZER))
if model.lexicon is not None:
write_zip(zf, os.path.join(tmpdir, LEXICON))
if hasattr(model, "bpe_codec"):
write_zip(zf, os.path.join(tmpdir, BPE_CODEC))
finally:
zf.close()
log.info("Finished saving model.")
atexit.register(lambda dir=tmpdir: shutil.rmtree(tmpdir))
def main(args):
sanity_check(args)
nmtrain.environment.init(args, nmtrain.enumeration.RunMode.TEST)
# Manager of batches and data
data_manager = nmtrain.data.DataManager()
# The model, chainer model inside
model = load_model(args)
# The watcher, who logs everything
watcher = nmtrain.TestWatcher(state=nmtrain.model.TestState(),
src_vocab=model.src_vocab,
trg_vocab=model.trg_vocab,
output_stream=sys.stdout)
# Classifier, that run the data and the model
classifier = nmtrain.classifiers.RNN_NMT()
print("~~ Testing ~~", file=sys.stderr)
print("gen_limit:", args.gen_limit, file=sys.stderr)
print("beam:", args.beam, file=sys.stderr)
print("word_penalty:", args.word_penalty, file=sys.stderr)
print("gpu:", args.gpu, file=sys.stderr)
log.info("Loading Data")
data_manager.load_test(src=args.src,
src_voc=model.src_vocab,
trg_voc=model.trg_vocab,
ref=args.ref,
bpe_codec=model.bpe_codec)
log.info("Loading Finished.")
# Begin Testing
tester = nmtrain.Tester(data=data_manager,
watcher=watcher,
trg_vocab=model.trg_vocab,
classifier=classifier,
predict=True,
eval_ppl=(args.ref is not None))
if model.__class__.__name__ == "NmtrainModel":
model = model.chainer_model
tester.test(model=model,
word_penalty=args.word_penalty,
beam_size=args.beam,
gen_limit=args.gen_limit)
def __init__(self, args):
# If init_model is provided, args will be overwritten
# Inside the constructor of NmtrainModel using the
# previous training specification.
self.nmtrain_model = nmtrain.NmtrainModel(args)
self.data_manager = nmtrain.data.DataManager()
# Training Parameters
self.maximum_epoch = args.epoch
self.bptt_len = args.bptt_len
self.early_stop_num = args.early_stop
self.save_models = args.save_models
# Unknown Trainers
self.unknown_trainer = nmtrain.data.unknown_trainer.from_string(
args.unknown_training)
# Location of output model
self.model_file = args.model_out
# SGD lr decay factor
self.sgd_lr_decay_factor = args.sgd_lr_decay_factor
self.sgd_lr_decay_after = args.sgd_lr_decay_after
# Testing configuration
self.test_beam = args.test_beam
self.test_word_penalty = args.test_word_penalty
self.test_gen_limit = args.test_gen_limit
# Load in the real data
log.info("Loading Data")
self.data_manager.load_train(src=args.src,
trg=args.trg,
src_voc=self.nmtrain_model.src_vocab,
trg_voc=self.nmtrain_model.trg_vocab,
src_dev=args.src_dev,
trg_dev=args.trg_dev,
src_test=args.src_test,
trg_test=args.trg_test,
batch_size=args.batch,
unk_cut=args.unk_cut,
src_max_vocab=args.src_max_vocab,
trg_max_vocab=args.trg_max_vocab,
max_sent_length=args.max_sent_length,
sort_method=args.sort_method,
batch_strategy=args.batch_strategy,
unknown_trainer=self.unknown_trainer,
bpe_codec=self.nmtrain_model.bpe_codec)
log.info("Loading Finished.")
# Finalize the model, according to the data
self.nmtrain_model.finalize_model()
def begin_evaluation(self):
self.dev_loss = 0
self.dev_loss_ctr = 0
self.predictions = []
log.info("Begin Evaluation...")
def temp_model_file(name="nmtrain.model"):
model_file = tempfile.gettempdir() + "/" + name
log.info("Model file is requested at:", model_file)
return model_file
