def save_sequences(opt, sequences, avg_scores, indiv_scores, l, split, full, data_loader): # This seems a bit roundabout since l = opt.train.dynamic in train.py # But it's in case we start checkpointing outside of epoch boundaries opt.train.dynamic.epoch = l if cfg.save: if full: names = {"gens": "gens", "scores": "scores", "indiv": "indiv.scores"} else: names = {"gens": "gens.small", "scores": "scores.small", "indiv": "indiv.scores.small"} # Save generated sequences data.save_eval_file(opt, sequences, names["gens"], split) if avg_scores is not None: # Save average scores over evaluation set for generated sequences # Scores computed are the ones the generator was initialized with data.save_eval_file(opt, avg_scores, names["scores"], split) if split == "dev": # Save individual scores data.save_eval_file( opt, indiv_scores, names["indiv"], split)
def log_losses(self, opt, losses): if (not cfg.toy and cfg.save) or cfg.test_save: data.save_eval_file(opt, losses["train"], "losses", split="train") data.save_eval_file(opt, losses['dev'], "losses", split="dev") data.save_eval_file(opt, losses['test'], "losses", split="test")
model = models.make_model(opt, n_vocab, n_ctx, n_special, load=False) print("Loading Weights") models.load_state_dict(model, model_file["state_dict"]) print("Done Loading Weights") model.eval() # Initialize variable for # of examples to cycle through data.set_max_sizes(data_loader, force_split=split) evaluator = evaluate.make_evaluator(opt, model, data_loader) evaluator.batch_variables["split"] = split model.cuda(cfg.device) loss = evaluator.epoch(opt, model, data_loader, split) data.save_eval_file(opt, loss, "losses", split=split) loss_str = [] loss_str.append("Per Token Loss: {}".format(loss["total_micro"])) loss_str.append("Per Token Perplexity: {}".format(loss["ppl_micro"])) loss_str.append("Per Example Loss: {}".format(loss["total_macro"])) loss_str.append("Per Example Perplexity: {}".format(loss["ppl_macro"])) loss_str = "\n".join(loss_str) print(loss_str) data.save_eval_file(opt, loss_str, "losses", split=split, ext="txt")