def __load_configs(self):
config_file = self.home / _CONFIG_FILE_NAME
if config_file.exists():
with open(str(config_file)) as f:
config = util.yaml_load(f.read())
if config is None:
config = {}
else:
with monit.section('Creating a .labml config'):
from uuid import uuid1
config = {'uuid': uuid1().hex}
with open(str(config_file), 'w') as f:
f.write(util.yaml_dump(config))
default_config = self.__default_config()
for k, v in default_config.items():
if k not in config:
config[k] = v
self.uuid = config['uuid']
web_api_url = config['web_api']
if web_api_url[0:4] != 'http':
web_api_url = f"https://api.lab-ml.com/api/v1/computer?labml_token={web_api_url}&"
self.web_api = WebAPIConfigs(
url=web_api_url,
frequency=config['web_api_frequency'],
verify_connection=config['web_api_verify_connection'],
open_browser=config['web_api_open_browser'])
def __init__(self, uuid: str):
runs = RunsSet()
self.run_info, self.name = runs.get(uuid)
with open(str(self.run_info.indicators_path), 'r') as f:
indicators = util.yaml_load(f.read())
if 'indicators' not in indicators:
raise RuntimeError("This run is corrupted or from an old version of LabML. "
"Please update labml_dashboard and run it on this project. "
"It will automatically migrate all the experiments.")
indicators = indicators['indicators']
inds = []
for k, v in indicators.items():
cn = v['class_name']
class_ = None
if cn == 'Histogram':
class_ = IndicatorClass.histogram
elif cn == 'Queue':
class_ = IndicatorClass.queue
elif cn == 'IndexedScalar':
class_ = IndicatorClass.scalar
elif cn == 'Scalar':
class_ = IndicatorClass.scalar
elif cn == 'Tensor':
class_ = IndicatorClass.tensor
if class_ is None:
continue
inds.append(Indicator(k, class_, self.run_info.uuid, v))
self.indicators = IndicatorCollection(inds)
def load_configs(configs_path: Path):
if not configs_path.exists():
return None
with open(str(configs_path), 'r') as file:
configs = util.yaml_load(file.read())
return configs
def get(self, uuid: str) -> Tuple[RunInfo, str]:
run_path = self._runs[uuid][0]
run_info_path = run_path / 'run.yaml'
with open(str(run_info_path), 'r') as f:
data = util.yaml_load(f.read())
run = RunInfo.from_dict(run_path.parent, data)
return run, self._runs[uuid][1]
def set_token(self, token: str):
with monit.section('Update ~/labml/configs.yaml'):
with open(str(self.configs_file)) as f:
config = util.yaml_load(f.read())
assert config is not None
config['web_api'] = token
with open(str(self.configs_file), 'w') as f:
f.write(util.yaml_dump(config))
def __load_configs(self):
if self.config_folder.is_file():
self.config_folder.unlink()
if not self.config_folder.exists():
self.config_folder.mkdir(parents=True)
if not self.projects_folder.exists():
self.projects_folder.mkdir()
if not self.app_folder.exists():
self.app_folder.mkdir()
if not self.runs_cache.exists():
self.runs_cache.mkdir()
if self.configs_file.exists():
with open(str(self.configs_file)) as f:
config = util.yaml_load(f.read())
if config is None:
config = {}
else:
logger.log([('~/labml/configs.yaml', Text.value),
' does not exist. Creating ',
(str(self.configs_file), Text.meta)])
config = {}
if 'uuid' not in config:
from uuid import uuid1
config['uuid'] = uuid1().hex
with open(str(self.configs_file), 'w') as f:
f.write(util.yaml_dump(config))
default_config = self.__default_config()
for k, v in default_config.items():
if k not in config:
config[k] = v
self.uuid = config['uuid']
web_api_url = config['web_api']
if web_api_url[0:4] != 'http':
web_api_url = f"https://api.labml.ai/api/v1/computer?labml_token={web_api_url}&"
self.web_api = WebAPIConfigs(
url=web_api_url,
frequency=config['web_api_frequency'],
verify_connection=config['web_api_verify_connection'],
open_browser=config['web_api_open_browser'],
is_default=web_api_url == self.__default_config()['web_api'])
self.web_api_sync = config['web_api_sync']
self.web_api_polling = config['web_api_polling']
self.tensorboard_port = config['tensorboard_port']
self.tensorboard_visible_port = config['tensorboard_visible_port']
self.tensorboard_host = config['tensorboard_host']
self.tensorboard_protocol = config['tensorboard_protocol']
def __init__(self, uuid: str):
runs = RunsSet()
self.run_info, self.name = runs.get(uuid)
with open(str(self.run_info.indicators_path), 'r') as f:
indicators = util.yaml_load(f.read())
inds = []
for k, v in indicators.items():
cn = v['class_name']
class_ = None
if cn == 'Histogram':
class_ = IndicatorClass.histogram
elif cn == 'Queue':
class_ = IndicatorClass.queue
elif cn == 'IndexedScalar':
class_ = IndicatorClass.scalar
elif cn == 'Scalar':
class_ = IndicatorClass.scalar
if class_ is None:
continue
inds.append(Indicator(k, class_, self.run_info.uuid))
with open(str(self.run_info.artifacts_path), 'r') as f:
artifacts = util.yaml_load(f.read())
for k, v in artifacts.items():
cn = v['class_name']
class_ = None
if cn == 'Tensor':
class_ = IndicatorClass.tensor
if class_ is None:
continue
inds.append(Indicator(k, class_, self.run_info.uuid))
self.indicators = IndicatorCollection(inds)
def load_cache(self):
if not self.cache_path.exists():
return
with open(str(self.cache_path), 'r') as f:
data = util.yaml_load(f.read())
if not data or data['path'] != str(self.path):
return
self.complete = data['complete']
self.size = data['size']
self.size_tensorboard = data['size_tensorboard']
self.size_checkpoints = data['size_checkpoints']
def __load_configs(self):
self.config_folder = self.home / CONFIGS_FOLDER
if self.config_folder.is_file():
self.config_folder.unlink()
if not self.config_folder.exists():
self.config_folder.mkdir(parents=True)
configs_file = self.config_folder / 'configs.yaml'
if configs_file.exists():
with open(str(configs_file)) as f:
config = util.yaml_load(f.read())
if config is None:
config = {}
else:
logger.log([('~/labml/configs.yaml', Text.value),
' does not exist. Creating ',
(str(configs_file), Text.meta)])
config = {}
if 'uuid' not in config:
from uuid import uuid1
config['uuid'] = uuid1().hex
with open(str(configs_file), 'w') as f:
f.write(util.yaml_dump(config))
default_config = self.__default_config()
for k, v in default_config.items():
if k not in config:
config[k] = v
self.uuid = config['uuid']
web_api_url = config['web_api']
if web_api_url[0:4] != 'http':
web_api_url = f"https://api.lab-ml.com/api/v1/computer?labml_token={web_api_url}&"
self.web_api = WebAPIConfigs(
url=web_api_url,
frequency=config['web_api_frequency'],
verify_connection=config['web_api_verify_connection'],
open_browser=config['web_api_open_browser'])
def __load_config_files(path: Path):
configs = []
while path.exists():
if path.is_dir():
config_file = path / _CONFIG_FILE_NAME
if config_file.is_file():
with open(str(config_file)) as f:
config = util.yaml_load(f.read())
if config is None:
config = {}
config['config_file_path'] = path
configs.append(config)
if str(path) == path.root:
break
path = path.parent
return configs
