def _capture(args: List[str]):
api_caller = ApiCaller("https://api.lab-ml.com/api/v1/track?",
{'run_uuid': generate_uuid()})
api_logs = ApiLogs()
data = {'name': 'Capture', 'comment': ' '.join(args), 'time': time.time()}
def _started(url):
if url is None:
return None
logger.log([('Monitor experiment at ', Text.meta), (url, Text.link)])
webbrowser.open(url)
api_caller.has_data(SimpleApiDataSource(data, callback=_started))
api_logs.set_api(api_caller, frequency=0)
thread = ExecutorThread(' '.join(args), api_logs)
thread.start()
thread.join()
data = {
'rank': 0,
'status': 'completed',
'details': None,
'time': time.time()
}
api_caller.has_data(
SimpleApiDataSource({
'status': data,
'time': time.time()
}))
api_caller.stop()
def _capture(args: List[str]):
api_caller = ApiCaller("https://api.labml.ai/api/v1/track?", {'run_uuid': generate_uuid()},
timeout_seconds=120)
api_logs = ApiLogs()
data = {
'name': 'Capture',
'comment': ' '.join(args),
'time': time.time()
}
api_caller.add_handler(ApiUrlHandler(True, 'Monitor output at '))
api_caller.has_data(SimpleApiDataSource(data))
api_logs.set_api(api_caller, frequency=0)
logger.log('Start capturing...', Text.meta)
if args:
thread = ExecutorThread(' '.join(args), api_logs)
thread.start()
thread.join()
else:
buffer = ''
stdin = sys.stdin
while stdin.readable():
data = stdin.read(1)
if len(data) == 0:
break
print(data, end='')
buffer += data
if '\n' in buffer or len(buffer) > 100:
api_logs.outputs(stdout_=buffer)
buffer = ''
if len(buffer) > 0:
api_logs.outputs(stdout_=buffer)
data = {
'rank': 0,
'status': 'completed',
'details': None,
'time': time.time()
}
api_caller.has_data(SimpleApiDataSource({
'status': data,
'time': time.time()
}))
api_caller.stop()
def _start_tracker(self):
tracker().reset_writers()
if self.is_evaluate:
return
if self.distributed_rank != 0:
return
if 'screen' in self.writers:
from labml.internal.tracker.writers import screen
tracker().add_writer(screen.ScreenWriter())
if 'sqlite' in self.writers:
from labml.internal.tracker.writers import sqlite
tracker().add_writer(
sqlite.Writer(self.run.sqlite_path, self.run.artifacts_folder))
if 'tensorboard' in self.writers:
from labml.internal.tracker.writers import tensorboard
tracker().add_writer(
tensorboard.Writer(self.run.tensorboard_log_path))
if 'wandb' in self.writers:
from labml.internal.tracker.writers import wandb
self.wandb = wandb.Writer()
tracker().add_writer(self.wandb)
else:
self.wandb = None
if 'comet' in self.writers:
from labml.internal.tracker.writers import comet
self.comet = comet.Writer()
tracker().add_writer(self.comet)
else:
self.comet = None
if 'file' in self.writers:
from labml.internal.tracker.writers import file
tracker().add_writer(file.Writer(self.run.log_file))
if 'web_api' in self.writers:
web_api_conf = lab_singleton().web_api
if web_api_conf is not None:
from labml.internal.tracker.writers import web_api
from labml.internal.api import ApiCaller
from labml.internal.api.experiment import ApiExperiment
api_caller = ApiCaller(web_api_conf.url,
{'run_uuid': self.run.uuid},
timeout_seconds=120)
self.web_api = ApiExperiment(
api_caller,
frequency=web_api_conf.frequency,
open_browser=web_api_conf.open_browser)
tracker().add_writer(
web_api.Writer(api_caller,
frequency=web_api_conf.frequency))
else:
self.web_api = None
def __init__(self, session_uuid: str, open_browser):
api_caller = ApiCaller(computer_singleton().web_api.url,
{'computer_uuid': computer_singleton().uuid, 'session_uuid': session_uuid},
timeout_seconds=120,
daemon=True)
self.writer = Writer(api_caller, frequency=computer_singleton().web_api.frequency)
self.header = Header(api_caller,
frequency=computer_singleton().web_api.frequency,
open_browser=open_browser)
self.scanner = Scanner()
def __init__(self):
api_caller = ApiCaller(computer_singleton().web_api.url,
{'computer_uuid': computer_singleton().uuid},
15)
self.writer = Writer(api_caller,
frequency=computer_singleton().web_api.frequency)
self.header = Header(
api_caller,
frequency=computer_singleton().web_api.frequency,
open_browser=computer_singleton().web_api.open_browser)
self.data = {}
def __init__(self, session_uuid: str):
api_caller = ApiCaller(computer_singleton().web_api.url,
{'computer_uuid': computer_singleton().uuid, 'session_uuid': session_uuid},
timeout_seconds=15,
daemon=True)
self.writer = Writer(api_caller, frequency=computer_singleton().web_api.frequency)
self.header = Header(api_caller,
frequency=computer_singleton().web_api.frequency,
open_browser=computer_singleton().web_api.open_browser)
self.data = {}
self.cache = {}
self.nvml = None
self.n_gpu = 0
try:
from py3nvml import py3nvml as nvml
self.nvml = nvml
except ImportError:
labml_notice('Install py3nvml to monitor GPUs:\n pip install py3nvml',
is_warn=False)