def __init__(self, cfg: Config): self._train = True self.log_interval = cfg.log_interval self.log_dir = cfg.run_dir self._img_log_dir = cfg.img_log_dir # get git commit hash if folder is a git repository current_dir = str(Path(__file__).absolute().parent) if subprocess.call(["git", "-C", current_dir, "branch"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0: git_output = subprocess.check_output( ["git", "-C", current_dir, "describe", "--always"]) cfg.update_config( {'commit_hash': git_output.strip().decode('ascii')}) # Additionally, the package version is stored in the config cfg.update_config({"package_version": __version__}) # store a copy of the config into the run folder cfg.dump_config(folder=self.log_dir) self.epoch = 0 self.update = 0 self._metrics = defaultdict(list) self.writer = None
def continue_run(run_dir: Path, config_file: Path = None, gpu: int = None): """Continue model training. Parameters ---------- run_dir : Path Path to the run directory. config_file : Path, optional Path to an additional config file. Each config argument in this file will overwrite the original run config. gpu : int, optional GPU id to use. Will override config argument 'device'. A value smaller than zero indicates CPU. """ # load config from base run and overwrite all elements with an optional new config base_config = Config(run_dir / "config.yml") if config_file is not None: base_config.update_config(config_file) base_config.is_continue_training = True # check if a GPU has been specified as command line argument. If yes, overwrite config if gpu is not None and gpu >= 0: config.device = f"cuda:{gpu}" if gpu is not None and gpu < 0: config.device = "cpu" start_training(base_config)
def finetune(config_file: Path = None, gpu: int = None): """Finetune a pre-trained model. Parameters ---------- config_file : Path, optional Path to an additional config file. Each config argument in this file will overwrite the original run config. The config file for finetuning must contain the argument `base_run_dir`, pointing to the folder of the pre-trained model, as well as 'finetune_modules' to indicate which model parts will be trained during fine-tuning. gpu : int, optional GPU id to use. Will override config argument 'device'. A value smaller than zero indicates CPU. """ # load finetune config and check for a non-empty list of finetune_modules temp_config = Config(config_file) if not temp_config.finetune_modules: raise ValueError("For finetuning, at least one model part has to be specified by 'finetune_modules'.") # extract base run dir, load base run config and combine with the finetune arguments config = Config(temp_config.base_run_dir / "config.yml") config.update_config({'run_dir': None, 'experiment_name': None}) config.update_config(config_file) config.is_finetuning = True # if the base run was a continue_training run, we need to override the continue_training flag from its config. config.is_continue_training = False # check if a GPU has been specified as command line argument. If yes, overwrite config if gpu is not None and gpu >= 0: config.device = f"cuda:{gpu}" if gpu is not None and gpu < 0: config.device = "cpu" start_training(config)
def finetune(config_file: Path = None, gpu: int = None): """Finetune a pre-trained model. Parameters ---------- config_file : Path, optional Path to an additional config file. Each config argument in this file will overwrite the original run config. The config file for finetuning must contain the argument `base_run_dir`, pointing to the folder of the pre-trained model. gpu : int, optional GPU id to use. Will override config argument 'device'. """ # load finetune config, extract base run dir, load base run config and combine with the finetune arguments temp_config = Config(config_file) config = Config(temp_config.base_run_dir / "config.yml") config.force_update({'run_dir': None, 'experiment_name': None}) config.update_config(config_file) config.is_finetuning = True # check if a GPU has been specified as command line argument. If yes, overwrite config if gpu is not None: config.device = f"cuda:{gpu}" start_training(config)
def create_config_files(base_config_path: Path, modify_dict: Dict[str, list], output_dir: Path): """Create configs, given a base config and a dictionary of parameters to modify. This function will create one config file for each combination of parameters defined in the modify_dict. Parameters ---------- base_config_path : Path Path to a base config file (.yml) modify_dict : dict Dictionary, mapping from parameter names to lists of possible parameter values. output_dir : Path Path to a folder where the generated configs will be stored """ if not output_dir.is_dir(): output_dir.mkdir(parents=True) # load base config as dictionary base_config = Config(base_config_path) experiment_name = base_config.experiment_name option_names = list(modify_dict.keys()) # iterate over each possible combination of hyper parameters for i, options in enumerate( itertools.product(*[val for val in modify_dict.values()])): base_config.update_config(dict(zip(option_names, options))) # create a unique run name name = experiment_name for key, val in zip(option_names, options): name += f"_{key}{val}" base_config.update_config({"experiment_name": name}) base_config.dump_config(output_dir, f"config_{i+1}.yml") print(f"Finished. Configs are stored in {output_dir}")
def __init__(self, cfg: Config): self._train = True self.log_interval = cfg.log_interval self.log_dir = cfg.run_dir self._img_log_dir = cfg.img_log_dir # get git commit hash if folder is a git repository cfg.update_config({'commit_hash': get_git_hash()}) # save git diff to file if branch is dirty if cfg.save_git_diff: save_git_diff(cfg.run_dir) # Additionally, the package version is stored in the config cfg.update_config({"package_version": __version__}) # store a copy of the config into the run folder cfg.dump_config(folder=self.log_dir) self.epoch = 0 self.update = 0 self._metrics = defaultdict(list) self.writer = None