def main(args):
from guild import tensorboard
tensorboard.setup_logging()
with util.TempDir("guild-tensorboard-", keep=args.keep_logdir) as tmp:
logdir = tmp.path
(log.info if args.keep_logdir else log.debug)("Using logdir %s",
logdir)
monitor = tensorboard.RunsMonitor(logdir, _list_runs_cb(args),
args.refresh_interval)
cli.out("Preparing runs for TensorBoard")
monitor.run_once(exit_on_error=True)
monitor.start()
try:
tensorboard.serve_forever(logdir=logdir,
host=(args.host or "0.0.0.0"),
port=(args.port or util.free_port()),
reload_interval=args.refresh_interval,
ready_cb=_open_cb(args))
except tensorboard.TensorboardError as e:
cli.error(str(e))
finally:
log.debug("Stopping")
monitor.stop()
log.debug("Removing logdir %s", logdir) # Handled by ctx mgr
if util.PLATFORM != "Windows":
cli.out()
def run_op(self, opspec, flags, restart, no_wait, stage, **opts):
with util.TempDir(prefix="guild-remote-stage-") as tmp:
if not restart:
op_src = _op_src(opspec)
if op_src:
_build_package(op_src, tmp.path)
remote_run_dir = self._init_remote_run(tmp.path, opspec, restart)
run_id = os.path.basename(remote_run_dir)
self._start_op(remote_run_dir, opspec, restart, flags, run_id, stage,
**opts)
if stage:
log.info("%s staged as on %s as %s", opspec, self.name, run_id)
log.info(
"To start the operation, use 'guild run -r %s --start %s'",
self.name,
run_id,
)
if no_wait or stage:
return run_id
try:
self._watch_started_op(remote_run_dir)
except KeyboardInterrupt:
raise remotelib.RemoteProcessDetached(run_id)
else:
return run_id
def _print_trials_cmd(batch_run, trials):
from guild.commands import run_impl
for trial in trials:
with util.TempDir() as tmp:
run = init_trial_run(batch_run, trial, tmp.path)
run_impl.run(restart=run.dir, print_cmd=True)
def _export_runs_to_gist_archives(runs, gist_repo):
with util.TempDir("guild-runs-export-") as tmp:
archives = [_run_export_archive(run, tmp.path) for run in runs]
_export_runs(zip(runs, archives))
for archive_src in archives:
archive_dest = os.path.join(gist_repo, os.path.basename(archive_src))
util.ensure_deleted(archive_dest)
shutil.move(archive_src, archive_dest)
def _handle_default(hiplot, data):
with util.TempDir("guild-compare-") as tmp:
csv_path = _hiplot_data_path(tmp.path)
_write_hiplot_data(data, csv_path)
html_path = _hiplot_html_path(tmp.path)
_generate_hiplot_html(hiplot, csv_path, html_path)
cli.out("Opening %s" % html_path, err=True)
util.open_url(html_path)
cli.out("To return to the prompt, press Ctrl-C", err=True)
_wait_forever()
def run_op(self, opspec, flags, restart, no_wait, **opts):
with util.TempDir(prefix="guild-remote-pkg-") as dist_dir:
_build_package(dist_dir)
remote_run_dir = self._init_remote_run(dist_dir, opspec, restart)
self._start_op(remote_run_dir, opspec, flags, **opts)
run_id = os.path.basename(remote_run_dir)
if no_wait:
return run_id
try:
self._watch_started_op(remote_run_dir)
except KeyboardInterrupt:
raise remotelib.RemoteProcessDetached(run_id)
else:
return run_id
def main(args):
tensorboard = _load_guild_tensorboard_module()
with util.TempDir("guild-tensorboard-") as logdir:
log.debug("Using logdir %s", logdir)
monitor = RunsMonitor(logdir, args)
monitor.start()
tensorboard.main(logdir=logdir,
host=(args.host or ""),
port=(args.port or util.free_port()),
reload_interval=args.refresh_interval,
ready_cb=(_open_url if not args.no_open else None))
log.debug("Stopping")
monitor.stop()
log.debug("Removing logdir %s", logdir) # Handled by ctx mgr
cli.out()
def _run_tensorboard(args):
from guild import tensorboard
tensorboard.setup_logging()
with util.TempDir("guild-tensorboard-", keep=_keep_logdir(args)) as tmp:
logdir = tmp.path
(log.info if args.keep_logdir else log.debug)("Using logdir %s", logdir)
tensorboard_options = _tensorboard_options(args)
monitor = tensorboard.RunsMonitor(
logdir,
_list_runs_cb(args),
interval=args.refresh_interval,
log_images=not args.skip_images,
log_hparams=not args.skip_hparams,
run_name_cb=_run_name_cb(args),
)
t0 = time.time()
cli.out("Preparing runs for TensorBoard")
monitor.run_once(exit_on_error=True)
if args.test_logdir:
cli.out("Initialized log dir %s" % logdir)
return
_maybe_log_prepare_time(t0)
monitor.start()
try:
tensorboard.serve_forever(
logdir=logdir,
host=(args.host or "0.0.0.0"),
port=(args.port or util.free_port()),
reload_interval=args.refresh_interval,
tensorboard_options=tensorboard_options,
ready_cb=_open_cb(args),
)
except tensorboard.TensorboardError as e:
cli.error(str(e))
finally:
log.debug("Stopping")
monitor.stop()
if not args.keep_logdir:
# Removal of temp logdir occurs when context manager
# exits.
log.debug("Removing logdir %s", logdir)
else:
print("TensorBoard logs saved in %s" % logdir)
if util.get_platform() != "Windows":
cli.out()
def _view_files(args):
with util.TempDir("guild-view-") as logdir:
log.debug("Using logdir %s", logdir)
list_runs_cb = lambda: runs_impl.runs_for_args(args)
monitor = run_util.RunsMonitor(
list_runs_cb, logdir, VIEW_FILES_REFRESH_INTERVAL)
monitor.start()
click.launch(logdir)
print("Monitoring runs at %s (Press CTRL+C to quit)" % logdir)
try:
util.wait_forever()
except KeyboardInterrupt:
pass
log.debug("Stopping")
monitor.stop()
log.debug("Removing logdir %s", logdir) # Handled by ctx mgr
if util.PLATFORM != "Windows":
cli.out()
def main(args):
tensorboard = _load_guild_tensorboard_module()
tensorboard.setup_logging()
with util.TempDir("guild-tensorboard-") as logdir:
log.debug("Using logdir %s", logdir)
list_runs_cb = lambda: runs_impl.runs_for_args(args)
monitor = tensorboard.RunsMonitor(list_runs_cb, logdir,
args.refresh_interval)
monitor.start()
try:
tensorboard.serve_forever(
logdir=logdir,
host=(args.host or "0.0.0.0"),
port=(args.port or util.free_port()),
reload_interval=args.refresh_interval,
ready_cb=(_open_url if not args.no_open else None))
except tensorboard.TensorboardError as e:
cli.error(str(e))
finally:
log.debug("Stopping")
monitor.stop()
log.debug("Removing logdir %s", logdir) # Handled by ctx mgr
if util.PLATFORM != "Windows":
cli.out()
def _handle_html_env(hiplot, data, html_path):
with util.TempDir("guild-compare-") as tmp:
csv_path = _hiplot_data_path(tmp.path)
_write_hiplot_data(data, csv_path)
_generate_hiplot_html(hiplot, csv_path, html_path)
cli.out("Saved HiPlot HTML to %s" % html_path, err=True)
def _install_external(name, dist_spec):
with util.TempDir(prefix="pip-", suffix="-download") as tmp:
wheel_path = _pip_wheel(name, dist_spec, tmp)
_install_external_wheel(wheel_path)
util.touch(_external_marker(name))
def _print_trial_cmd(trial):
with util.TempDir("guild-trial-") as run_dir:
trial.init(run_dir, quiet=True)
trial.run(print_cmd=True, quiet=True)
def _replace_run(archive, run_id, dest_dir):
with util.TempDir("guild-gist-run-") as tmp:
_extract_archive(archive, tmp.path)
extracted_run_dir = _validate_extracted_run(tmp.path, run_id, archive)
dest_run_dir = os.path.join(dest_dir, run_id)
_replace_run_dir(dest_run_dir, extracted_run_dir)
def _run_tmp_batch(S, extra_env):
assert S.batch_op
with util.TempDir() as tmp:
_init_batch_run(S, tmp.path)
_run_op(S.batch_op, S.args, extra_env)
def _print_trials_cmd(batch_run, trials):
for trial in trials:
with util.TempDir() as tmp:
run = _init_trial_run(batch_run, trial, tmp.path)
run_impl.run(restart=run.dir, print_cmd=True)
def _run_batch_tmp_with_env(op, cmd_env, args):
with util.TempDir() as tmp:
op.set_run_dir(tmp.path)
op.batch_op.cmd_env.update(cmd_env)
_init_batch_run(op)
_run_op(op.batch_op, args)
def _install_external(name, dist_spec):
tmp = util.TempDir(prefix="pip-", suffix="-download")
wheel_path = _pip_wheel(name, dist_spec, tmp.path)
_install_external_wheel(wheel_path)
util.touch(_external_marker(name))
tmp.delete()
def _print_trial_cmd(trial):
with util.TempDir("guild-trial-") as tmp:
trial.init(tmp.path, quiet=True)
trial.run(print_cmd=True, quiet=True)
