def save_sequences(opt, sequences, avg_scores, indiv_scores, l, split, full,
data_loader):
# This seems a bit roundabout since l = opt.train.dynamic in train.py
# But it's in case we start checkpointing outside of epoch boundaries
opt.train.dynamic.epoch = l
if cfg.save:
if full:
names = {
"gens": "gens",
"scores": "scores",
"indiv": "indiv.scores"
}
else:
names = {
"gens": "gens.small",
"scores": "scores.small",
"indiv": "indiv.scores.small"
}
# Save generated sequences
data.save_eval_file(opt, sequences, names["gens"], split)
if avg_scores is not None:
# Save average scores over evaluation set for generated sequences
# Scores computed are the ones the generator was initialized with
data.save_eval_file(opt, avg_scores, names["scores"], split)
if split == "dev":
# Save individual scores
data.save_eval_file(opt, indiv_scores, names["indiv"], split)
model = models.make_model(opt, n_vocab, n_ctx, n_special, load=False)
print("Loading Weights")
models.load_state_dict(model, model_file["state_dict"])
print("Done Loading Weights")
model.eval()
# Initialize variable for # of examples to cycle through
data.set_max_sizes(data_loader, force_split=split)
evaluator = evaluate.make_evaluator(opt, model, data_loader)
evaluator.batch_variables["split"] = split
model.cuda(cfg.device)
loss = evaluator.epoch(opt, model, data_loader, split)
data.save_eval_file(opt, loss, "losses", split=split)
loss_str = []
loss_str.append("Per Token Loss: {}".format(loss["total_micro"]))
loss_str.append("Per Token Perplexity: {}".format(loss["ppl_micro"]))
loss_str.append("Per Example Loss: {}".format(loss["total_macro"]))
loss_str.append("Per Example Perplexity: {}".format(loss["ppl_macro"]))
loss_str = "\n".join(loss_str)
print(loss_str)
data.save_eval_file(opt, loss_str, "losses", split=split, ext="txt")
def log_losses(self, opt, losses):
if (not cfg.toy and cfg.save) or cfg.test_save:
data.save_eval_file(opt, losses["train"], "losses", split="train")
data.save_eval_file(opt, losses['dev'], "losses", split="dev")
data.save_eval_file(opt, losses['test'], "losses", split="test")
