def start(self, runs: List[Path]):
if self.pipe is not None:
self.pipe.kill()
self._create_symlink_folder()
for p in runs:
run = RunInfo.from_path(p)
os.symlink(run.tensorboard_log_path, self.symlink_path / run.uuid)
self.pipe = Popen([
'tensorboard', f'--logdir={self.symlink_path}', '--port',
f'{self.port}', '--bind_all'
],
env=os.environ.copy(),
stderr=PIPE)
time.sleep(5)
output = ''.join(get_output(self.pipe.stderr))
if output.find('Press CTRL+C to quit') != -1:
logger.log([('Tensorboard: ', Text.meta), (output, Text.subtle)])
return True, output
else:
logger.log([('Failed to start Tensorboard: ', Text.warning),
(output, Text.subtle)])
return False, output
def get(self, uuid: str) -> Tuple[RunInfo, str]:
run_path = self._runs[uuid][0]
run_info_path = run_path / 'run.yaml'
with open(str(run_info_path), 'r') as f:
data = util.yaml_load(f.read())
run = RunInfo.from_dict(run_path.parent, data)
return run, self._runs[uuid][1]
def scan(self):
run = RunInfo.from_path(self.path)
self.size = self._folder_size(run.run_path)
self.size_tensorboard = self._folder_size(run.tensorboard_log_path)
self.size_checkpoints = self._folder_size(run.checkpoint_path)
if run.run_log_path.exists():
with open(str(run.run_log_path), 'r') as f:
lines = f.read().split('\n')
for l in lines:
if not l:
continue
d = json.loads(l)
if d['status'] in COMPLETED_STATUSES:
self.complete = True
self.save_cache()
def start(self, runs: List[Path]):
if self.pipe is not None:
self.pipe.kill()
self._create_symlink_folder()
for p in runs:
run = RunInfo.from_path(p)
os.symlink(run.tensorboard_log_path, self.symlink_path / run.uuid)
self.pipe = Popen([
'tensorboard', f'--logdir={self.symlink_path}', '--port',
f'{self.port}', '--bind_all'
],
env=os.environ.copy(),
stderr=PIPE)
output = self.pipe.stderr.readline().decode('utf-8')
if output.find('Press CTRL+C to quit') != -1:
logger.log([('Tensorboard: ', Text.meta), (output, Text.subtle)])
return True
else:
return False
def get(self, uuid: str) -> Tuple[RunInfo, str]:
run = RunInfo.from_path(self._runs[uuid][0])
return run, self._runs[uuid][1]