def save_checkpoint(self,
logdir,
checkpoint,
is_best,
save_n_best=5,
main_metric="loss",
minimize_metric=True):
suffix = f"{checkpoint['stage']}.{checkpoint['epoch']}"
filepath = UtilsFactory.save_checkpoint(
logdir=f"{logdir}/checkpoints/",
checkpoint=checkpoint,
suffix=suffix,
is_best=is_best,
is_last=True)
checkpoint_metric = checkpoint["valid_metrics"].get(main_metric, None)
checkpoint_metric = checkpoint_metric or checkpoint.get("epoch", -1)
self.top_best_metrics.append((filepath, checkpoint_metric))
self.top_best_metrics = sorted(self.top_best_metrics,
key=lambda x: x[1],
reverse=not minimize_metric)
if len(self.top_best_metrics) > save_n_best:
last_item = self.top_best_metrics.pop(-1)
last_filepath = last_item[0]
os.remove(last_filepath)
def save(self):
if self.epoch % self.save_period == 0:
checkpoint = self.algorithm.prepare_checkpoint()
checkpoint["epoch"] = self.epoch
filename = UtilsFactory.save_checkpoint(logdir=self.logdir,
checkpoint=checkpoint,
suffix=str(self.epoch))
print("Checkpoint saved to: %s" % filename)
def _save_checkpoint(self):
if self.epoch % self.save_period == 0:
checkpoint = self.algorithm.pack_checkpoint()
checkpoint["epoch"] = self.epoch
filename = UtilsFactory.save_checkpoint(
logdir=self.logdir,
checkpoint=checkpoint,
suffix=str(self.epoch)
)
print(f"Checkpoint saved to: {filename}")
def save_checkpoint(
self,
logdir,
checkpoint,
is_best,
):
suffix = f"{checkpoint['stage']}.iter.{self.count}"
filepath = UtilsFactory.save_checkpoint(
logdir=f"{logdir}/checkpoints/",
checkpoint=checkpoint,
suffix=suffix,
is_best=is_best,
is_last=True)
print(f"\nSaved checkpoint at {filepath}")