def save_checkpoint(self, logdir: str, checkpoint: Dict, is_best: bool, save_n_best: int = 5, main_metric: str = "loss", minimize_metric: bool = True): suffix = f"{checkpoint['stage']}.{checkpoint['epoch']}" filepath = utils.save_checkpoint(logdir=f"{logdir}/checkpoints/", checkpoint=checkpoint, suffix=suffix, is_best=is_best, is_last=True) valid_metrics = checkpoint["valid_metrics"] checkpoint_metric = valid_metrics[main_metric] self.top_best_metrics.append( (filepath, checkpoint_metric, valid_metrics)) self.top_best_metrics = sorted(self.top_best_metrics, key=lambda x: x[1], reverse=not minimize_metric) if len(self.top_best_metrics) > save_n_best: last_item = self.top_best_metrics.pop(-1) last_filepath = last_item[0] os.remove(last_filepath) checkpoints = [(Path(filepath).stem, valid_metric) for (filepath, _, valid_metric) in self.top_best_metrics ] best_valid_metrics = checkpoints[0][1] metrics = OrderedDict([("best", best_valid_metrics)] + checkpoints + [("last", valid_metrics)]) safitty.save(metrics, f"{logdir}/checkpoints/_metrics.json")
def dump_environment( experiment_config: Dict, logdir: str, configs_path: List[str] = None, ) -> None: """ Saves config, environment variables and package list in JSON into logdir Args: experiment_config (dict): experiment config logdir (str): path to logdir configs_path: path(s) to config """ configs_path = configs_path or [] configs_path = [ Path(path) for path in configs_path if isinstance(path, str) ] config_dir = Path(logdir) / "configs" config_dir.mkdir(exist_ok=True, parents=True) environment = get_environment_vars() safitty.save(experiment_config, config_dir / "_config.json") safitty.save(environment, config_dir / "_environment.json") pip_pkg = list_pip_packages() (config_dir / "pip-packages.txt").write_text(pip_pkg) conda_pkg = list_conda_packages() if conda_pkg: (config_dir / "conda-packages.txt").write_text(conda_pkg) for path in configs_path: name: str = path.name outpath = config_dir / name shutil.copyfile(path, outpath) config_str = json.dumps(experiment_config, indent=2, ensure_ascii=False) config_str = config_str.replace("\n", "\n\n") environment_str = json.dumps(environment, indent=2, ensure_ascii=False) environment_str = environment_str.replace("\n", "\n\n") pip_pkg = pip_pkg.replace("\n", "\n\n") conda_pkg = conda_pkg.replace("\n", "\n\n") with SummaryWriter(config_dir) as writer: writer.add_text("_config", config_str, 0) writer.add_text("_environment", environment_str, 0) writer.add_text("pip-packages", pip_pkg, 0) if conda_pkg: writer.add_text("conda-packages", conda_pkg, 0)
def dump_config( experiment_config: Dict, logdir: str, configs_path: List[str] = None, ) -> None: """ Saves config and environment in JSON into logdir Args: experiment_config (dict): experiment config logdir (str): path to logdir configs_path: path(s) to config """ configs_path = configs_path or [] configs_path = [ Path(path) for path in configs_path if isinstance(path, str) ] config_dir = Path(logdir) / "configs" config_dir.mkdir(exist_ok=True, parents=True) environment = get_environment_vars() safitty.save(experiment_config, config_dir / "_config.json") safitty.save(environment, config_dir / "_environment.json") for path in configs_path: name: str = path.name outpath = config_dir / name shutil.copyfile(path, outpath) config_str = json.dumps(experiment_config, indent=2) config_str = config_str.replace("\n", "\n\n") environment_str = json.dumps(environment, indent=2) environment_str = environment_str.replace("\n", "\n\n") with SummaryWriter(config_dir) as writer: writer.add_text("config", config_str, 0) writer.add_text("environment", environment_str, 0)
def save_metric(self, logdir: str, metrics: Dict) -> None: safitty.save(metrics, f"{logdir}/checkpoints/{self.metric_filename}")