def start(self,
*,
run: Optional[int] = None,
checkpoint: Optional[int] = None):
if run is not None:
with logger.section("Loading checkpoint"):
global_step = self._load_checkpoint(run, checkpoint)
if global_step is None:
logger.set_successful(False)
global_step = 0
else:
global_step = 0
self.run.start_step = global_step
logger.internal().set_start_global_step(global_step)
self.__print_info_and_check_repo()
self.configs_processor.print()
self.run.save_info()
if self.configs_processor is not None:
self.configs_processor.save(self.run.configs_path)
logger.internal().save_indicators(self.run.indicators_path)
with open(str(self.run.diff_path), "w") as f:
f.write(self.run.diff)
def start(self,
*,
run_uuid: Optional[str] = None,
checkpoint: Optional[int] = None):
if run_uuid is not None:
if checkpoint is None:
checkpoint = -1
global_step = self.__start_from_checkpoint(run_uuid, checkpoint)
else:
global_step = 0
self.run.start_step = global_step
logger.internal().set_start_global_step(global_step)
self.__print_info_and_check_repo()
if self.configs_processor is not None:
self.configs_processor.print()
self.run.save_info()
if self.configs_processor is not None:
self.configs_processor.save(self.run.configs_path)
logger.internal().save_indicators(self.run.indicators_path)
logger.internal().save_artifacts(self.run.artifacts_path)
def __init__(self, *,
name: Optional[str],
python_file: Optional[str],
comment: Optional[str],
writers: Set[str] = None):
"""
### Create the experiment
:param name: name of the experiment
:param python_file: `__file__` that invokes this. This is stored in
the experiments list.
:param comment: a short description of the experiment
The experiments log keeps track of `python_file`, `name`, `comment` as
well as the git commit.
Experiment maintains the locations of checkpoints, logs, etc.
"""
if python_file is None:
python_file = self.__get_caller_file()
if python_file.startswith('<ipython'):
assert is_ipynb()
if name is None:
raise ValueError("You must specify python_file or experiment name"
" when creating an experiment from a python notebook.")
self.lab = Lab(os.getcwd())
python_file = 'notebook.ipynb'
else:
self.lab = Lab(python_file)
if name is None:
file_path = pathlib.PurePath(python_file)
name = file_path.stem
logger.internal().set_data_path(self.lab.data_path)
if comment is None:
comment = ''
self.name = name
self.experiment_path = self.lab.experiments / name
self.check_repo_dirty = self.lab.check_repo_dirty
self.configs_processor = None
experiment_path = pathlib.Path(self.experiment_path)
if not experiment_path.exists():
experiment_path.mkdir(parents=True)
self.run = Run.create(
experiment_path=self.experiment_path,
python_file=python_file,
trial_time=time.localtime(),
comment=comment)
repo = git.Repo(self.lab.path)
self.run.commit = repo.head.commit.hexsha
self.run.commit_message = repo.head.commit.message.strip()
self.run.is_dirty = repo.is_dirty()
self.run.diff = repo.git.diff()
checkpoint_saver = self._create_checkpoint_saver()
logger.internal().set_checkpoint_saver(checkpoint_saver)
if writers is None:
writers = {'sqlite', 'tensorboard'}
if 'sqlite' in writers:
logger.internal().add_writer(sqlite.Writer(self.run.sqlite_path))
if 'tensorboard' in writers:
logger.internal().add_writer(tensorboard.Writer(self.run.tensorboard_log_path))
logger.internal().set_numpy_path(self.run.numpy_path)