def set_tags(selected):
for run in selected:
old_tags = run.get("tags")
run.write_attr("tags", _tags_for_run(old_tags, args))
if args.sync_labels:
run.write_attr("label",
_synced_label_for_tags(run, old_tags, args))
cli.out("Modified tags for %i run(s)" % len(selected), err=True)
def _print_run_info_ordered(data):
for name, val in data:
if isinstance(val, list):
_print_run_info_list(name, val)
elif isinstance(val, dict):
_print_run_info_dict(name, val)
else:
cli.out("%s: %s" % (name, val))
def _check_needed_matching_runs(op):
matching = _find_matching_runs(op)
if matching:
cli.out(
"Skipping because the following runs match "
"this operation (--needed specified):")
_print_matching_runs(matching)
raise SystemExit(0)
def _check_needed_restart(op, args):
assert os.path.exists(op.run_dir), op.run_dir
run_id = os.path.basename(op.run_dir)
assert run_id.startswith(args.restart), (run_id, args.restart)
run = runlib.Run(run_id, op.run_dir)
if not op_util.restart_needed(run, op.flag_vals):
cli.out("Skipping run because flags have not changed " "(--needed specified)")
raise SystemExit(0)
def _print_stage_pending_instructions(op):
run_id = op.run_id
cli.out(
"{op} staged as {run_id}\n"
"To start the operation, use 'guild run --start {run_id}'".format(
op=op.opdef.fullname, run_id=run_id
)
)
def _print_or_save_one_trial(op, args):
flag_vals = op.opdef.flag_values(include_none=True)
if args.print_trials:
op_util.print_trials([flag_vals])
else:
assert args.save_trials
cli.out("Saving 1 trial to %s" % args.save_trials)
op_util.save_trials([flag_vals], args.save_trials)
def _handle_run_failed(e, remote):
run_id = os.path.basename(e.remote_run_dir)
cli.out(
"Try 'guild runs info %s -O -r %s' to view its output." %
(run_id[:8], remote.name),
err=True,
)
cli.error()
def _upgrade_pip(config):
cmd_args = [_pip_bin(config.env_dir), "install", "--upgrade", "pip"]
cli.out("Upgrading pip")
log.debug("pip upgrade cmd: %s", cmd_args)
try:
subprocess.check_output(cmd_args)
except subprocess.CalledProcessError as e:
cli.error(str(e), exit_status=e.returncode)
def _print_cmd(op, args):
if op.batch_op:
formatted = " ".join(_preview_cmd(op.batch_op))
cli.out(formatted)
_print_batch_trials_cmd(op, args)
else:
formatted = " ".join(_preview_cmd(op))
cli.out(formatted)
def _write_csv(args):
data = get_data(args, format_cells=False, skip_header_if_empty=True) or []
with _open_file(args.csv) as out:
writer = csv.writer(out, lineterminator="\n")
for row in data:
writer.writerow(row)
if args.csv != "-":
cli.out("Wrote %i row(s) to %s" % (len(data) - 1, args.csv), err=True)
def one_run(args, ctx):
filtered = filtered_runs(args, ctx=ctx)
if not filtered:
cli.out("No matching runs", err=True)
cli.error()
runspec = args.run or "1"
selected = select_runs(filtered, [runspec], ctx)
return cmd_impl_support.one_run(selected, runspec, ctx)
def _init_guild_env(config):
cli.out("Initializing Guild environment in {}".format(config.env_dir))
try:
init.init_env(config.env_dir, config.guild_home, config.isolate_resources)
except init.PermissionError as e:
cli.error("unable to write to %s - do you have write permission?" % e.args[0])
except init.InitError as e:
cli.error(e)
def _print_scalars(args):
runs = runs_impl.runs_for_args(args)
index = indexlib.RunIndex()
index.refresh(runs, ["scalar"])
for run in runs:
cli.out("[%s] %s" % (run.short_id, run_util.format_op_desc(run)))
for s in index.run_scalars(run):
cli.out(" %s" % run_util.run_scalar_key(s))
def _print_staged_dir_instructions(op):
cmd = " ".join(_preview_cmd(op))
cli.out(
"{op} staged in {dir}\n"
"To start the operation, use "
"'(cd {dir} && source .guild/ENV && {cmd})'".format(
op=op.opdef.fullname, dir=op.run_dir, cmd=cmd
)
)
def _print_scalars(args):
runs = runs_impl.runs_for_args(args)
index = indexlib.RunIndex()
index.refresh(runs, ["scalar"])
for run in runs:
cli.out("[%s] %s" % (run.short_id, run_util.format_operation(run, nowarn=True)))
for s in index.run_scalars(run):
key, step, val = (run_util.run_scalar_key(s), s["last_step"], s["last_val"])
cli.out(" %s: %f (step %i)" % (key, val, step))
def _try_stop_remote_run(run, remote_lock, no_wait):
try:
plugin = guild.plugin.for_name(remote_lock.plugin_name)
except LookupError:
log.warning("error syncing run '%s': plugin '%s' not available",
run.id, remote_lock.plugin_name)
else:
cli.out("Stopping %s (remote)" % run.id)
plugin.stop_run(run, dict(no_wait=no_wait))
def _notify_newer_version():
cli.out(
click.style(
"A newer version of Guild AI is available. Run "
"'pip install guildai --upgrade' to install it.",
bold=True,
),
err=True,
)
def _sync_files(self):
job_dir = self.run.get("cloudml-job-dir")
if not job_dir:
self.log.error(
"cloudml-job-dir not defined for run %s, cannot sync files",
self.run.id)
return
cli.out("Synchronizing job output for run %s" % self.run.id)
self._rsync_files(job_dir, self.run.path)
def _write_completion_script_to_user_config(shell, script):
path = os.path.join(config.user_config_home(), "%s_completion" % shell)
cli.out("Writing completion script to %s" % util.format_dir(path),
err=True)
util.ensure_dir(os.path.dirname(path))
with open(path, "w") as f:
f.write(script)
f.write("\n")
return path
def package_info(args):
for i, (project, pkg) in enumerate(_iter_project_names(args.packages)):
if i > 0:
cli.out("---")
exit_status = pip_util.print_package_info(
project, verbose=args.verbose, show_files=args.files
)
if exit_status != 0:
log.warning("unknown package %s", pkg)
def _backup_init_script(init_script):
if os.path.exists(init_script):
backup = _init_script_backup(init_script)
cli.out(
"Backing up %s to %s" %
(util.format_dir(init_script), util.format_dir(backup)),
err=True,
)
shutil.copy(init_script, backup)
def _print_guild_latest_versions(check):
if check.offline:
cli.out("latest_guild_version: unchecked (offline)")
else:
cur_ver = guild.__version__
latest_ver = _latest_version(check)
latest_ver_desc = latest_ver or "unknown (error)"
cli.out("latest_guild_version: %s" % latest_ver_desc)
if latest_ver:
check.newer_version_available = _is_newer(latest_ver, cur_ver)
def _remote_op(op, prompt, default_resp, args):
if not args.yes:
cli.out(prompt)
if args.yes or cli.confirm("Continue?", default_resp):
try:
op()
except guild.remote.OperationNotSupported as e:
cli.error(e)
except guild.remote.OperationError as e:
cli.error(e)
def _maybe_sync_run(run):
if run.status != "running":
log.debug("run %s status is '%s', skipping sync", run.id, run.status)
return
log.debug("getting remote run %s", run.id)
remote_run = _remote_run_for_local_run(run)
log.debug("remote run for %s: %s", run.id, _remote_run_desc(remote_run))
if remote_run:
cli.out("Syncing %s" % run.id)
_sync_remote_run(remote_run)
def set_labels(selected):
for run in selected:
if args.clear:
run.del_attr("label")
else:
run.write_attr("label", _label_for_run(run, args).strip())
if args.clear:
cli.out("Cleared label for %i run(s)" % len(selected))
else:
cli.out("Labeled %i run(s)" % len(selected))
def _handle_tensorflow_import_error(e):
if "tensorflow" in str(e):
cli.out(
"TensorFlow is not installed.\n"
"Refer to https://www.tensorflow.org/install/ for help "
"installing TensorFlow on your system.",
err=True)
else:
cli.out("Error loading TensorBoard: %s" % e, err=True)
cli.error()
def _handle_default(hiplot, data):
with util.TempDir("guild-compare-") as tmp:
csv_path = _hiplot_data_path(tmp.path)
_write_hiplot_data(data, csv_path)
html_path = _hiplot_html_path(tmp.path)
_generate_hiplot_html(hiplot, csv_path, html_path)
cli.out("Opening %s" % html_path, err=True)
util.open_url(html_path)
cli.out("To return to the prompt, press Ctrl-C", err=True)
_wait_forever()
def compare_runs(get_data_cb):
cli.out("Preparing data for compare", err=True)
hiplot = _hiplot_include_exe()
data = get_data_cb()
_format_data_for_hiplot(data)
html_path_env = os.getenv("HIPLOT_HTML")
if html_path_env:
_handle_html_env(hiplot, data, html_path_env)
else:
_handle_default(hiplot, data)
def _init_venv(config):
cmd_args = _venv_cmd_args(config)
if not cmd_args:
cli.out("Skipping virtual env")
return
cli.out("Creating virtual environment")
log.debug("venv args: %s", cmd_args)
try:
subprocess.check_call(cmd_args)
except subprocess.CalledProcessError as e:
cli.error(str(e), exit_status=e.returncode)
def _print_path(path, args):
if args.full_path:
path = os.path.abspath(path)
elif args.sourcecode:
path = _sourcecode_rel_path(path)
if not path:
return
if args.no_format or args.full_path:
cli.out(path)
else:
cli.out(" %s" % path)
