def calc_configs_dict(self,
configs: Dict[str, any],
configs_override: Optional[Dict[str, any]]):
self.configs_processor = ConfigProcessorDict(configs, configs_override)
self.configs_processor()
logger.log()
def calc_configs(self,
configs: Configs,
configs_override: Optional[Dict[str, any]],
run_order: Optional[List[Union[List[str], str]]]):
self.configs_processor = ConfigProcessor(configs, configs_override)
self.configs_processor(run_order)
logger.log()
def calc_configs(self, configs: Configs,
configs_override: Optional[Dict[str, any]],
run_order: Optional[List[Union[List[str], str]]]):
if configs_override is None:
configs_override = {}
if global_params_singleton().configs is not None:
configs_override.update(global_params_singleton().configs)
self.configs_processor = ConfigProcessor(configs, configs_override)
self.configs_processor(run_order)
logger.log()
from labml.internal.configs.processor_dict import ConfigProcessorDict
processor = ConfigProcessorDict({
'cnn_size': 10,
'batch_size': 12
}, {'batch_size': 2})
processor()
processor.print()
class Experiment:
r"""
Each experiment has different configurations or algorithms.
An experiment can have multiple runs.
Keyword Arguments:
name (str, optional): name of the experiment
python_file (str, optional): path of the Python file that
created the experiment
comment (str, optional): a short description of the experiment
writers (Set[str], optional): list of writers to write stat to
ignore_callers: (Set[str], optional): list of files to ignore when
automatically determining ``python_file``
tags (Set[str], optional): Set of tags for experiment
"""
run: Run
configs_processor: Optional[ConfigProcessor]
# whether not to start the experiment if there are uncommitted changes.
check_repo_dirty: bool
checkpoint_saver: CheckpointSaver
def __init__(self, *,
name: Optional[str],
python_file: Optional[str],
comment: Optional[str],
writers: Set[str],
ignore_callers: Set[str],
tags: Optional[Set[str]]):
if python_file is None:
python_file = get_caller_file(ignore_callers)
if python_file.startswith('<ipython'):
assert is_ipynb()
if name is None:
raise ValueError("You must specify python_file or experiment name"
" when creating an experiment from a python notebook.")
lab_singleton().set_path(os.getcwd())
python_file = 'notebook.ipynb'
else:
lab_singleton().set_path(python_file)
if name is None:
file_path = pathlib.PurePath(python_file)
name = file_path.stem
if comment is None:
comment = ''
self.name = name
self.experiment_path = lab_singleton().experiments / name
self.check_repo_dirty = lab_singleton().check_repo_dirty
self.configs_processor = None
experiment_path = pathlib.Path(self.experiment_path)
if not experiment_path.exists():
experiment_path.mkdir(parents=True)
if tags is None:
tags = set(name.split('_'))
self.run = Run.create(
experiment_path=self.experiment_path,
python_file=python_file,
trial_time=time.localtime(),
comment=comment,
tags=list(tags))
repo = git.Repo(lab_singleton().path)
self.run.commit = repo.head.commit.hexsha
self.run.commit_message = repo.head.commit.message.strip()
self.run.is_dirty = repo.is_dirty()
self.run.diff = repo.git.diff()
logger_internal().reset_writers()
if 'sqlite' in writers:
from labml.internal.logger.writers import sqlite
artifacts_folder = pathlib.Path(self.run.artifacts_folder)
if not artifacts_folder.exists():
artifacts_folder.mkdir(parents=True)
logger_internal().add_writer(
sqlite.Writer(self.run.sqlite_path, self.run.artifacts_folder))
if 'tensorboard' in writers:
from labml.internal.logger.writers import tensorboard
logger_internal().add_writer(tensorboard.Writer(self.run.tensorboard_log_path))
self.checkpoint_saver = None
def __print_info_and_check_repo(self):
"""
🖨 Print the experiment info and check git repo status
"""
logger.log()
logger.log([
(self.name, Text.title),
': ',
(str(self.run.uuid), Text.meta)
])
if self.run.comment != '':
logger.log(['\t', (self.run.comment, Text.highlight)])
logger.log([
"\t"
"[dirty]" if self.run.is_dirty else "[clean]",
": ",
(f"\"{self.run.commit_message.strip()}\"", Text.highlight)
])
if self.run.load_run is not None:
logger.log([
"\t"
"loaded from",
": ",
(f"{self.run.load_run}", Text.meta2),
])
# Exit if git repository is dirty
if self.check_repo_dirty and self.run.is_dirty:
logger.log([("[FAIL]", Text.danger),
" Cannot trial an experiment with uncommitted changes."])
exit(1)
def _load_checkpoint(self, checkpoint_path: pathlib.PurePath):
if self.checkpoint_saver is not None:
self.checkpoint_saver.load(checkpoint_path)
def save_checkpoint(self):
if self.checkpoint_saver is not None:
self.checkpoint_saver.save(logger_internal().global_step)
def calc_configs(self,
configs: Configs,
configs_override: Optional[Dict[str, any]],
run_order: Optional[List[Union[List[str], str]]]):
self.configs_processor = ConfigProcessor(configs, configs_override)
self.configs_processor(run_order)
logger.log()
def calc_configs_dict(self,
configs: Dict[str, any],
configs_override: Optional[Dict[str, any]]):
self.configs_processor = ConfigProcessorDict(configs, configs_override)
self.configs_processor()
logger.log()
def __start_from_checkpoint(self, run_uuid: str, checkpoint: Optional[int]):
checkpoint_path, global_step = experiment_run.get_last_run_checkpoint(
self.experiment_path,
run_uuid,
checkpoint)
if global_step is None:
return 0
else:
with monit.section("Loading checkpoint"):
self._load_checkpoint(checkpoint_path)
self.run.load_run = run_uuid
return global_step
def start(self, *,
run_uuid: Optional[str] = None,
checkpoint: Optional[int] = None):
if run_uuid is not None:
if checkpoint is None:
checkpoint = -1
global_step = self.__start_from_checkpoint(run_uuid, checkpoint)
else:
global_step = 0
self.run.start_step = global_step
logger_internal().set_start_global_step(global_step)
self.__print_info_and_check_repo()
if self.configs_processor is not None:
self.configs_processor.print()
self.run.save_info()
if self.configs_processor is not None:
self.configs_processor.save(self.run.configs_path)
logger_internal().save_indicators(self.run.indicators_path)
logger_internal().save_artifacts(self.run.artifacts_path)
if self.configs_processor:
logger_internal().write_h_parameters(self.configs_processor.get_hyperparams())
class Experiment:
r"""
Each experiment has different configurations or algorithms.
An experiment can have multiple runs.
Keyword Arguments:
name (str, optional): name of the experiment
python_file (str, optional): path of the Python file that
created the experiment
comment (str, optional): a short description of the experiment
writers (Set[str], optional): list of writers to write stat to
ignore_callers: (Set[str], optional): list of files to ignore when
automatically determining ``python_file``
tags (Set[str], optional): Set of tags for experiment
"""
run: Run
configs_processor: Optional[ConfigProcessor]
# whether not to start the experiment if there are uncommitted changes.
check_repo_dirty: bool
checkpoint_saver: CheckpointSaver
def __init__(self, *,
name: Optional[str],
python_file: Optional[str],
comment: Optional[str],
writers: Set[str],
ignore_callers: Set[str],
tags: Optional[Set[str]]):
if python_file is None:
python_file = get_caller_file(ignore_callers)
if python_file.startswith('<ipython'):
assert is_ipynb()
if name is None:
raise ValueError("You must specify python_file or experiment name"
" when creating an experiment from a python notebook.")
lab_singleton().set_path(os.getcwd())
python_file = 'notebook.ipynb'
else:
lab_singleton().set_path(python_file)
if name is None:
file_path = pathlib.PurePath(python_file)
name = file_path.stem
if comment is None:
comment = ''
if global_params_singleton().comment is not None:
comment = global_params_singleton().comment
self.name = name
self.experiment_path = lab_singleton().experiments / name
self.check_repo_dirty = lab_singleton().check_repo_dirty
self.configs_processor = None
experiment_path = pathlib.Path(self.experiment_path)
if not experiment_path.exists():
experiment_path.mkdir(parents=True)
if tags is None:
tags = set(name.split('_'))
self.run = Run.create(
experiment_path=self.experiment_path,
python_file=python_file,
trial_time=time.localtime(),
comment=comment,
tags=list(tags))
try:
repo = git.Repo(lab_singleton().path)
self.run.commit = repo.head.commit.hexsha
self.run.commit_message = repo.head.commit.message.strip()
self.run.is_dirty = repo.is_dirty()
self.run.diff = repo.git.diff()
except git.InvalidGitRepositoryError:
if not is_colab() and not is_kaggle():
warnings.warn(f"Not a valid git repository",
UserWarning, stacklevel=4)
self.run.commit = 'unknown'
self.run.commit_message = ''
self.run.is_dirty = True
self.run.diff = ''
logger_internal().reset_writers()
if 'sqlite' in writers:
from labml.internal.logger.writers import sqlite
logger_internal().add_writer(
sqlite.Writer(self.run.sqlite_path, self.run.artifacts_folder))
if 'tensorboard' in writers:
from labml.internal.logger.writers import tensorboard
logger_internal().add_writer(tensorboard.Writer(self.run.tensorboard_log_path))
if 'web_api' in writers:
from labml.internal.logger.writers import web_api
self.web_api = web_api.Writer()
logger_internal().add_writer(self.web_api)
else:
self.web_api = None
self.checkpoint_saver = CheckpointSaver(self.run.checkpoint_path)
def __print_info(self):
"""
🖨 Print the experiment info and check git repo status
"""
logger.log()
logger.log([
(self.name, Text.title),
': ',
(str(self.run.uuid), Text.meta)
])
if self.run.comment != '':
logger.log(['\t', (self.run.comment, Text.highlight)])
commit_message = self.run.commit_message.strip().replace('\n', '¶ ').replace('\r', '')
logger.log([
"\t"
"[dirty]" if self.run.is_dirty else "[clean]",
": ",
(f"\"{commit_message}\"", Text.highlight)
])
if self.run.load_run is not None:
logger.log([
"\t"
"loaded from",
": ",
(f"{self.run.load_run}", Text.meta2),
])
def _load_checkpoint(self, checkpoint_path: pathlib.Path):
if not self.checkpoint_saver.load(checkpoint_path):
logger.log('No models registered', Text.warning)
def save_checkpoint(self):
self.checkpoint_saver.save(logger_internal().global_step)
def calc_configs(self,
configs: Configs,
configs_override: Optional[Dict[str, any]],
run_order: Optional[List[Union[List[str], str]]]):
if configs_override is None:
configs_override = {}
if global_params_singleton().configs is not None:
configs_override.update(global_params_singleton().configs)
self.configs_processor = ConfigProcessor(configs, configs_override)
self.configs_processor(run_order)
logger.log()
def calc_configs_dict(self,
configs: Dict[str, any],
configs_override: Optional[Dict[str, any]]):
self.configs_processor = ConfigProcessorDict(configs, configs_override)
self.configs_processor()
logger.log()
def __start_from_checkpoint(self, run_uuid: str, checkpoint: Optional[int]):
checkpoint_path, global_step = experiment_run.get_run_checkpoint(
run_uuid,
checkpoint)
if global_step is None:
return 0
else:
with monit.section("Loading checkpoint"):
self._load_checkpoint(checkpoint_path)
self.run.load_run = run_uuid
return global_step
def load_models(self, *,
models: List[str],
run_uuid: Optional[str] = None,
checkpoint: Optional[int] = None):
if checkpoint is None:
checkpoint = -1
checkpoint_path, global_step = experiment_run.get_run_checkpoint(
run_uuid,
checkpoint)
if global_step is None:
warnings.warn(f"Could not find checkpoint",
UserWarning, stacklevel=4)
return
with monit.section("Loading checkpoint"):
_ = self.checkpoint_saver.load(checkpoint_path, models)
def start(self, *,
run_uuid: Optional[str] = None,
checkpoint: Optional[int] = None):
if run_uuid is not None:
if checkpoint is None:
checkpoint = -1
global_step = self.__start_from_checkpoint(run_uuid, checkpoint)
else:
global_step = 0
self.run.start_step = global_step
logger_internal().set_start_global_step(global_step)
self.__print_info()
if self.check_repo_dirty and self.run.is_dirty:
logger.log([("[FAIL]", Text.danger),
" Cannot trial an experiment with uncommitted changes."])
exit(1)
if self.configs_processor is not None:
self.configs_processor.print()
self.run.save_info()
if self.configs_processor is not None:
self.configs_processor.save(self.run.configs_path)
if self.web_api is not None:
self.web_api.set_info(run_uuid=self.run.uuid,
name=self.name,
comment=self.run.comment)
if self.configs_processor is not None:
self.web_api.set_configs(self.configs_processor.to_json())
self.web_api.start()
logger_internal().save_indicators(self.run.indicators_path)
if self.configs_processor:
logger_internal().write_h_parameters(self.configs_processor.get_hyperparams())
return ExperimentWatcher(self)
def finish(self, status: str, details: any = None):
with open(str(self.run.run_log_path), 'a') as f:
end_time = time.time()
data = json.dumps({'status': status,
'details': details,
'time': end_time}, indent=None)
f.write(data + '\n')
if self.web_api is not None:
self.web_api.status(status, details, end_time)