def start(self, *, run: Optional[int] = None, checkpoint: Optional[int] = None): if run is not None: with logger.section("Loading checkpoint"): global_step = self._load_checkpoint(run, checkpoint) if global_step is None: logger.set_successful(False) global_step = 0 else: global_step = 0 self.run.start_step = global_step logger.internal().set_start_global_step(global_step) self.__print_info_and_check_repo() self.configs_processor.print() self.run.save_info() if self.configs_processor is not None: self.configs_processor.save(self.run.configs_path) logger.internal().save_indicators(self.run.indicators_path) with open(str(self.run.diff_path), "w") as f: f.write(self.run.diff)
def start(self, *, run_uuid: Optional[str] = None, checkpoint: Optional[int] = None): if run_uuid is not None: if checkpoint is None: checkpoint = -1 global_step = self.__start_from_checkpoint(run_uuid, checkpoint) else: global_step = 0 self.run.start_step = global_step logger.internal().set_start_global_step(global_step) self.__print_info_and_check_repo() if self.configs_processor is not None: self.configs_processor.print() self.run.save_info() if self.configs_processor is not None: self.configs_processor.save(self.run.configs_path) logger.internal().save_indicators(self.run.indicators_path) logger.internal().save_artifacts(self.run.artifacts_path)
def __init__(self, *, name: Optional[str], python_file: Optional[str], comment: Optional[str], writers: Set[str] = None): """ ### Create the experiment :param name: name of the experiment :param python_file: `__file__` that invokes this. This is stored in the experiments list. :param comment: a short description of the experiment The experiments log keeps track of `python_file`, `name`, `comment` as well as the git commit. Experiment maintains the locations of checkpoints, logs, etc. """ if python_file is None: python_file = self.__get_caller_file() if python_file.startswith('<ipython'): assert is_ipynb() if name is None: raise ValueError("You must specify python_file or experiment name" " when creating an experiment from a python notebook.") self.lab = Lab(os.getcwd()) python_file = 'notebook.ipynb' else: self.lab = Lab(python_file) if name is None: file_path = pathlib.PurePath(python_file) name = file_path.stem logger.internal().set_data_path(self.lab.data_path) if comment is None: comment = '' self.name = name self.experiment_path = self.lab.experiments / name self.check_repo_dirty = self.lab.check_repo_dirty self.configs_processor = None experiment_path = pathlib.Path(self.experiment_path) if not experiment_path.exists(): experiment_path.mkdir(parents=True) self.run = Run.create( experiment_path=self.experiment_path, python_file=python_file, trial_time=time.localtime(), comment=comment) repo = git.Repo(self.lab.path) self.run.commit = repo.head.commit.hexsha self.run.commit_message = repo.head.commit.message.strip() self.run.is_dirty = repo.is_dirty() self.run.diff = repo.git.diff() checkpoint_saver = self._create_checkpoint_saver() logger.internal().set_checkpoint_saver(checkpoint_saver) if writers is None: writers = {'sqlite', 'tensorboard'} if 'sqlite' in writers: logger.internal().add_writer(sqlite.Writer(self.run.sqlite_path)) if 'tensorboard' in writers: logger.internal().add_writer(tensorboard.Writer(self.run.tensorboard_log_path)) logger.internal().set_numpy_path(self.run.numpy_path)