def end_evaluation(self, dev_data, trg_vocab): if len(self.predictions) != 0: self.state.bleu_scores.append(calculate_bleu(self.predictions, trg_dev, trg_vocab)) # Adding previous dev perplexities if available if len(self.state.dev_perplexities) > 0: prev_dev_ppl = self.state.dev_ppl() else: prev_dev_ppl = None # Update the dev perplexities self.state.dev_perplexities.append(math.exp(self.dev_loss / self.dev_loss_ctr)) # Generate one line report dev_ppl = self.state.dev_ppl() if prev_dev_ppl is None: dev_ppl_report = "DEV_PPL=%.3f" % dev_ppl else: dev_ppl_report = "DEV_PPL=(%.3f -> %.3f)" % (prev_dev_ppl, dev_ppl) # Appending BLEU scores information if generated if len(self.state.bleu_scores) != 0: dev_ppl_report += ", BLEU=%s" % self.state.bleu() # Reporting log.info("End Evaluation: %s" % (dev_ppl_report))
def begin_evaluation(self): log.info("Decoding Started") self.time = time.time() self.predictions = [] self.test_loss = 0 self.test_loss_ctr = 0 self.attentions = []
def end_epoch(self, new_data_arrangement): self.state.finished_epoch += 1 self.state.batch_indexes = new_data_arrangement self.state.time_spent.append(time.time() - self.time) self.state.perplexities.append(math.exp(self.epoch_loss / self.epoch_update_counter)) log.info("Epoch %d finished! ppl=%.3f, time=%.4f mins" % (self.state.finished_epoch, self.state.ppl(), self.state.last_time() / 60))
def batch_update(self, id, loss=0, batch_size=1, col_size=1): ppl = math.exp(float(loss)) self.epoch_loss += loss self.epoch_update_counter += 1 self.state.trained_sentence += batch_size log.info("[%d] Sentence trained: %d, Batch(PPL=%.3f, size=(%d,%d), wps=%d, id=%d)" % (self.state.finished_epoch + 1, self.state.trained_sentence, ppl, batch_size, col_size, abs((batch_size * col_size) / (time.time() - self.batch_time)), id))
def begin_epoch(self): # Number of training sentences self.trained = 0 # To measure ppl self.epoch_loss = 0 self.epoch_update_counter = 0 # To measure time self.time = time.time() # To measure number of trained sentence self.state.trained_sentence = 0 # Verbose log.info("Start Epoch %d" % (self.state.finished_epoch + 1))
def end_evaluation(self, test_data, trg_vocab): log.info("Decoding Finished, starting evaluation if reference is provided.") self.state.time_spent.append(time.time() - self.time) ref = test_data.trg_path if ref is not None: self.state.bleu_scores.append(calculate_bleu(self.predictions, ref, trg_vocab)) self.state.perplexities.append(math.exp(self.test_loss / self.test_loss_ctr)) # Creating evaluation string eval_string = "Time=%.2f mins" % (self.state.last_time() / 60) if ref is not None: eval_string += " " + ("BLEU=%s, test_ppl=%.3f" % (str(self.state.bleu()), self.state.ppl())) log.info("Evaluation Finished!", eval_string)
def save(model, out_file): if not out_file.endswith(".zip"): out_file = out_file + ".zip" tmpdir = tempfile.mkdtemp() log.info("Saving model to", out_file) # Saving Specification pickle_save(os.path.join(tmpdir, SPEC), model.specification.__dict__) # Saving vocabularies pickle_save(os.path.join(tmpdir, SRC_VOC), model.src_vocab) pickle_save(os.path.join(tmpdir, TRG_VOC), model.trg_vocab) # Saving optimizer state chainer.serializers.save_npz(os.path.join(tmpdir, OPTIMIZER), model.optimizer) # Saving training state pickle_save(os.path.join(tmpdir, STATE), model.training_state) # Saving Weight chainer.serializers.save_npz(os.path.join(tmpdir, WEIGHT), model.chainer_model) # Saving Lexicon if model.lexicon is not None: pickle_save(os.path.join(tmpdir, LEXICON), model.lexicon) # Saving BPE codec if hasattr(model, "bpe_codec"): pickle_save(os.path.join(tmpdir, BPE_CODEC), model.bpe_codec) # Zipping zf = zipfile.ZipFile(out_file, mode="w", compression=zipfile.ZIP_DEFLATED) try: write_zip(zf, os.path.join(tmpdir, SRC_VOC)) write_zip(zf, os.path.join(tmpdir, TRG_VOC)) write_zip(zf, os.path.join(tmpdir, STATE)) write_zip(zf, os.path.join(tmpdir, WEIGHT)) write_zip(zf, os.path.join(tmpdir, SPEC)) write_zip(zf, os.path.join(tmpdir, OPTIMIZER)) if model.lexicon is not None: write_zip(zf, os.path.join(tmpdir, LEXICON)) if hasattr(model, "bpe_codec"): write_zip(zf, os.path.join(tmpdir, BPE_CODEC)) finally: zf.close() log.info("Finished saving model.") atexit.register(lambda dir=tmpdir: shutil.rmtree(tmpdir))
def main(args): sanity_check(args) nmtrain.environment.init(args, nmtrain.enumeration.RunMode.TEST) # Manager of batches and data data_manager = nmtrain.data.DataManager() # The model, chainer model inside model = load_model(args) # The watcher, who logs everything watcher = nmtrain.TestWatcher(state=nmtrain.model.TestState(), src_vocab=model.src_vocab, trg_vocab=model.trg_vocab, output_stream=sys.stdout) # Classifier, that run the data and the model classifier = nmtrain.classifiers.RNN_NMT() print("~~ Testing ~~", file=sys.stderr) print("gen_limit:", args.gen_limit, file=sys.stderr) print("beam:", args.beam, file=sys.stderr) print("word_penalty:", args.word_penalty, file=sys.stderr) print("gpu:", args.gpu, file=sys.stderr) log.info("Loading Data") data_manager.load_test(src=args.src, src_voc=model.src_vocab, trg_voc=model.trg_vocab, ref=args.ref, bpe_codec=model.bpe_codec) log.info("Loading Finished.") # Begin Testing tester = nmtrain.Tester(data=data_manager, watcher=watcher, trg_vocab=model.trg_vocab, classifier=classifier, predict=True, eval_ppl=(args.ref is not None)) if model.__class__.__name__ == "NmtrainModel": model = model.chainer_model tester.test(model=model, word_penalty=args.word_penalty, beam_size=args.beam, gen_limit=args.gen_limit)
def __init__(self, args): # If init_model is provided, args will be overwritten # Inside the constructor of NmtrainModel using the # previous training specification. self.nmtrain_model = nmtrain.NmtrainModel(args) self.data_manager = nmtrain.data.DataManager() # Training Parameters self.maximum_epoch = args.epoch self.bptt_len = args.bptt_len self.early_stop_num = args.early_stop self.save_models = args.save_models # Unknown Trainers self.unknown_trainer = nmtrain.data.unknown_trainer.from_string( args.unknown_training) # Location of output model self.model_file = args.model_out # SGD lr decay factor self.sgd_lr_decay_factor = args.sgd_lr_decay_factor self.sgd_lr_decay_after = args.sgd_lr_decay_after # Testing configuration self.test_beam = args.test_beam self.test_word_penalty = args.test_word_penalty self.test_gen_limit = args.test_gen_limit # Load in the real data log.info("Loading Data") self.data_manager.load_train(src=args.src, trg=args.trg, src_voc=self.nmtrain_model.src_vocab, trg_voc=self.nmtrain_model.trg_vocab, src_dev=args.src_dev, trg_dev=args.trg_dev, src_test=args.src_test, trg_test=args.trg_test, batch_size=args.batch, unk_cut=args.unk_cut, src_max_vocab=args.src_max_vocab, trg_max_vocab=args.trg_max_vocab, max_sent_length=args.max_sent_length, sort_method=args.sort_method, batch_strategy=args.batch_strategy, unknown_trainer=self.unknown_trainer, bpe_codec=self.nmtrain_model.bpe_codec) log.info("Loading Finished.") # Finalize the model, according to the data self.nmtrain_model.finalize_model()
def begin_evaluation(self): self.dev_loss = 0 self.dev_loss_ctr = 0 self.predictions = [] log.info("Begin Evaluation...")
def temp_model_file(name="nmtrain.model"): model_file = tempfile.gettempdir() + "/" + name log.info("Model file is requested at:", model_file) return model_file