def new_run_process(python_file: PurePath, *,
configs: Optional[Dict[str, any]], comment: Optional[str]):
p = Process(python_file,
configs=configs,
comment=comment,
lab_path=lab.get_path())
p()
def get_predictor():
conf = Configs()
experiment.evaluate()
# This will download a pretrained model checkpoint and some cached files.
# It will download the archive as `saved_checkpoint.tar.gz` and extract it.
#
# If you have a locally trained model load it directly with
# run_uuid = 'RUN_UUID'
# And for latest checkpoint
# checkpoint = None
run_uuid, checkpoint = experiment.load_bundle(
lab.get_path() / 'saved_checkpoint.tar.gz',
url=
'https://github.com/lab-ml/python_autocomplete/releases/download/0.0.4/transformer_checkpoint.tar.gz'
)
conf_dict = experiment.load_configs(run_uuid)
experiment.configs(conf, conf_dict)
experiment.add_pytorch_models(get_modules(conf))
experiment.load(run_uuid, checkpoint)
experiment.start()
conf.model.eval()
return Predictor(conf.model, cache('stoi', lambda: conf.text.stoi),
cache('itos', lambda: conf.text.itos))
def load_experiment() -> Configs:
conf = Configs()
experiment.evaluate()
# This will download a pretrained model checkpoint and some cached files.
# It will download the archive as `saved_checkpoint.tar.gz` and extract it.
#
# If you have a locally trained model load it directly with
# run_uuid = 'RUN_UUID'
# And for latest checkpoint
# checkpoint = None
# run_uuid = 'a6cff3706ec411ebadd9bf753b33bae6' # bpe
# checkpoint = None
run_uuid, checkpoint = experiment.load_bundle(
lab.get_path() / 'saved_checkpoint.tar.gz',
url=
'https://github.com/lab-ml/python_autocomplete/releases/download/0.0.5/bundle.tar.gz'
)
conf_dict = experiment.load_configs(run_uuid)
conf_dict['text.is_load_data'] = False
experiment.configs(conf, conf_dict)
experiment.add_pytorch_models(get_modules(conf))
experiment.load(run_uuid, checkpoint)
experiment.start()
return conf
def track_disk():
res = psutil.disk_usage(lab.get_path())
tracker.add({
'disk.free': res.free,
'disk.total': res.total,
'disk.used': res.used,
})
def _load_module_main(python_file: PurePath):
module_path = python_file.relative_to(lab.get_path())
module_path = str(module_path).replace('/', '.').replace('.py', '')
experiment_module = importlib.import_module(module_path)
main_func = getattr(experiment_module, 'main', None)
if main_func is None:
raise ValueError(
'The experiment should have a function called main, that will be executed.'
)
return main_func
def new_run(python_file: PurePath,
*,
configs: Optional[Dict[str, any]] = None,
comment: Optional[str] = None,
lab_path: Optional[PurePath] = None):
if lab_path is None:
lab_path = lab.get_path()
module_path = python_file.relative_to(lab_path)
module_path = str(module_path).replace('/', '.').replace('.py', '')
_experiment.global_params_singleton().configs = configs
_experiment.global_params_singleton().comment = comment
experiment_module = importlib.import_module(module_path)
experiment_module.main()
def test_psutil():
# sudo apt-get install gcc python3-dev
# xcode on mac
# pip install psutil
import psutil
# https://psutil.readthedocs.io/en/latest/#
inspect(mac=psutil.MACOS, linux=psutil.LINUX, windows=psutil.WINDOWS)
inspect(psutil.net_io_counters()._asdict())
inspect(psutil.net_if_addrs())
inspect(psutil.net_if_stats())
inspect(psutil.virtual_memory()._asdict())
inspect(psutil.cpu_count())
inspect(psutil.cpu_times()._asdict())
inspect(psutil.cpu_stats()._asdict())
inspect(psutil.cpu_freq()._asdict())
inspect(psutil.cpu_percent(percpu=True))
inspect(psutil.disk_usage(lab.get_path())._asdict())
inspect(psutil.Process().as_dict())
inspect([p for p in psutil.process_iter()])
# inspect(psutil.Process().terminate())
# inspect('test')
p = psutil.Process()
with p.oneshot():
inspect(p.memory_info()._asdict())
inspect(p.memory_percent())
inspect(p.cpu_percent(1))
inspect(p.cpu_times()._asdict())
inspect(p.num_threads())
inspect(p.threads())
try:
inspect(p.cpu_num())
except AttributeError as e:
pass
try:
inspect(psutil.sensors_temperatures())
except AttributeError as e:
pass
try:
inspect(psutil.sensors_fans())
except AttributeError as e:
pass
try:
inspect(psutil.sensors_battery()._asdict())
except AttributeError as e:
pass
def _test():
from labml.internal.computer.configs import computer_singleton
from labml import lab
import time
tb = TensorBoardStarter(computer_singleton().tensorboard_symlink_dir)
# for k, v in os.environ.items():
# print(k, v)
res = tb.start([
lab.get_path() / 'logs' / 'sample' / '9f7970d6a98611ebbc6bacde48001122',
])
print(res)
time.sleep(100)
def _test():
from labml.internal.computer.configs import computer_singleton
from labml import lab
from labml.internal.lab import lab_singleton
import time
lab_singleton().set_path(
str(Path(os.path.abspath(__file__)).parent.parent.parent.parent))
tb = TensorBoardStarter(computer_singleton().tensorboard_symlink_dir)
# for k, v in os.environ.items():
# print(k, v)
res = tb.start([
lab.get_path() / 'logs' / 'sample' /
'68233e98cb5311eb9aa38d17b08f3a1d',
])
print(res)
time.sleep(100)
from labml import lab
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
options = Options()
options.headless = True
driver = webdriver.Firefox(executable_path=str(lab.get_path() / 'geckodriver'),
options=options)
def screenshot():
driver.get('http://localhost:3000/chart')
driver.save_screenshot('screenshot.png')
def screenshot_div():
from PIL import Image
driver.get(
'https://web.lab-ml.com/chart?run_uuid=6e8a1a44d21711ea8099f318d43ad04c'
)
element = WebDriverWait(driver, 10).until(
expected_conditions.presence_of_element_located(
(By.ID, "sample-chart")))
# element = driver.find_element_by_id("sample-chart")
location = element.location
size = element.size
from labml import experiment, lab
if __name__ == '__main__':
experiment.save_bundle(lab.get_path() / 'bundle.tar.gz',
'39b03a1e454011ebbaff2b26e3148b3d',
data_files=[
'cache/itos.json', 'cache/n_tokens.json',
'cache/stoi.json'
])
