def set_tags(selected): for run in selected: old_tags = run.get("tags") run.write_attr("tags", _tags_for_run(old_tags, args)) if args.sync_labels: run.write_attr("label", _synced_label_for_tags(run, old_tags, args)) cli.out("Modified tags for %i run(s)" % len(selected), err=True)
def _print_run_info_ordered(data): for name, val in data: if isinstance(val, list): _print_run_info_list(name, val) elif isinstance(val, dict): _print_run_info_dict(name, val) else: cli.out("%s: %s" % (name, val))
def _check_needed_matching_runs(op): matching = _find_matching_runs(op) if matching: cli.out( "Skipping because the following runs match " "this operation (--needed specified):") _print_matching_runs(matching) raise SystemExit(0)
def _check_needed_restart(op, args): assert os.path.exists(op.run_dir), op.run_dir run_id = os.path.basename(op.run_dir) assert run_id.startswith(args.restart), (run_id, args.restart) run = runlib.Run(run_id, op.run_dir) if not op_util.restart_needed(run, op.flag_vals): cli.out("Skipping run because flags have not changed " "(--needed specified)") raise SystemExit(0)
def _print_stage_pending_instructions(op): run_id = op.run_id cli.out( "{op} staged as {run_id}\n" "To start the operation, use 'guild run --start {run_id}'".format( op=op.opdef.fullname, run_id=run_id ) )
def _print_or_save_one_trial(op, args): flag_vals = op.opdef.flag_values(include_none=True) if args.print_trials: op_util.print_trials([flag_vals]) else: assert args.save_trials cli.out("Saving 1 trial to %s" % args.save_trials) op_util.save_trials([flag_vals], args.save_trials)
def _handle_run_failed(e, remote): run_id = os.path.basename(e.remote_run_dir) cli.out( "Try 'guild runs info %s -O -r %s' to view its output." % (run_id[:8], remote.name), err=True, ) cli.error()
def _upgrade_pip(config): cmd_args = [_pip_bin(config.env_dir), "install", "--upgrade", "pip"] cli.out("Upgrading pip") log.debug("pip upgrade cmd: %s", cmd_args) try: subprocess.check_output(cmd_args) except subprocess.CalledProcessError as e: cli.error(str(e), exit_status=e.returncode)
def _print_cmd(op, args): if op.batch_op: formatted = " ".join(_preview_cmd(op.batch_op)) cli.out(formatted) _print_batch_trials_cmd(op, args) else: formatted = " ".join(_preview_cmd(op)) cli.out(formatted)
def _write_csv(args): data = get_data(args, format_cells=False, skip_header_if_empty=True) or [] with _open_file(args.csv) as out: writer = csv.writer(out, lineterminator="\n") for row in data: writer.writerow(row) if args.csv != "-": cli.out("Wrote %i row(s) to %s" % (len(data) - 1, args.csv), err=True)
def one_run(args, ctx): filtered = filtered_runs(args, ctx=ctx) if not filtered: cli.out("No matching runs", err=True) cli.error() runspec = args.run or "1" selected = select_runs(filtered, [runspec], ctx) return cmd_impl_support.one_run(selected, runspec, ctx)
def _init_guild_env(config): cli.out("Initializing Guild environment in {}".format(config.env_dir)) try: init.init_env(config.env_dir, config.guild_home, config.isolate_resources) except init.PermissionError as e: cli.error("unable to write to %s - do you have write permission?" % e.args[0]) except init.InitError as e: cli.error(e)
def _print_scalars(args): runs = runs_impl.runs_for_args(args) index = indexlib.RunIndex() index.refresh(runs, ["scalar"]) for run in runs: cli.out("[%s] %s" % (run.short_id, run_util.format_op_desc(run))) for s in index.run_scalars(run): cli.out(" %s" % run_util.run_scalar_key(s))
def _print_staged_dir_instructions(op): cmd = " ".join(_preview_cmd(op)) cli.out( "{op} staged in {dir}\n" "To start the operation, use " "'(cd {dir} && source .guild/ENV && {cmd})'".format( op=op.opdef.fullname, dir=op.run_dir, cmd=cmd ) )
def _print_scalars(args): runs = runs_impl.runs_for_args(args) index = indexlib.RunIndex() index.refresh(runs, ["scalar"]) for run in runs: cli.out("[%s] %s" % (run.short_id, run_util.format_operation(run, nowarn=True))) for s in index.run_scalars(run): key, step, val = (run_util.run_scalar_key(s), s["last_step"], s["last_val"]) cli.out(" %s: %f (step %i)" % (key, val, step))
def _try_stop_remote_run(run, remote_lock, no_wait): try: plugin = guild.plugin.for_name(remote_lock.plugin_name) except LookupError: log.warning("error syncing run '%s': plugin '%s' not available", run.id, remote_lock.plugin_name) else: cli.out("Stopping %s (remote)" % run.id) plugin.stop_run(run, dict(no_wait=no_wait))
def _notify_newer_version(): cli.out( click.style( "A newer version of Guild AI is available. Run " "'pip install guildai --upgrade' to install it.", bold=True, ), err=True, )
def _sync_files(self): job_dir = self.run.get("cloudml-job-dir") if not job_dir: self.log.error( "cloudml-job-dir not defined for run %s, cannot sync files", self.run.id) return cli.out("Synchronizing job output for run %s" % self.run.id) self._rsync_files(job_dir, self.run.path)
def _write_completion_script_to_user_config(shell, script): path = os.path.join(config.user_config_home(), "%s_completion" % shell) cli.out("Writing completion script to %s" % util.format_dir(path), err=True) util.ensure_dir(os.path.dirname(path)) with open(path, "w") as f: f.write(script) f.write("\n") return path
def package_info(args): for i, (project, pkg) in enumerate(_iter_project_names(args.packages)): if i > 0: cli.out("---") exit_status = pip_util.print_package_info( project, verbose=args.verbose, show_files=args.files ) if exit_status != 0: log.warning("unknown package %s", pkg)
def _backup_init_script(init_script): if os.path.exists(init_script): backup = _init_script_backup(init_script) cli.out( "Backing up %s to %s" % (util.format_dir(init_script), util.format_dir(backup)), err=True, ) shutil.copy(init_script, backup)
def _print_guild_latest_versions(check): if check.offline: cli.out("latest_guild_version: unchecked (offline)") else: cur_ver = guild.__version__ latest_ver = _latest_version(check) latest_ver_desc = latest_ver or "unknown (error)" cli.out("latest_guild_version: %s" % latest_ver_desc) if latest_ver: check.newer_version_available = _is_newer(latest_ver, cur_ver)
def _remote_op(op, prompt, default_resp, args): if not args.yes: cli.out(prompt) if args.yes or cli.confirm("Continue?", default_resp): try: op() except guild.remote.OperationNotSupported as e: cli.error(e) except guild.remote.OperationError as e: cli.error(e)
def _maybe_sync_run(run): if run.status != "running": log.debug("run %s status is '%s', skipping sync", run.id, run.status) return log.debug("getting remote run %s", run.id) remote_run = _remote_run_for_local_run(run) log.debug("remote run for %s: %s", run.id, _remote_run_desc(remote_run)) if remote_run: cli.out("Syncing %s" % run.id) _sync_remote_run(remote_run)
def set_labels(selected): for run in selected: if args.clear: run.del_attr("label") else: run.write_attr("label", _label_for_run(run, args).strip()) if args.clear: cli.out("Cleared label for %i run(s)" % len(selected)) else: cli.out("Labeled %i run(s)" % len(selected))
def _handle_tensorflow_import_error(e): if "tensorflow" in str(e): cli.out( "TensorFlow is not installed.\n" "Refer to https://www.tensorflow.org/install/ for help " "installing TensorFlow on your system.", err=True) else: cli.out("Error loading TensorBoard: %s" % e, err=True) cli.error()
def _handle_default(hiplot, data): with util.TempDir("guild-compare-") as tmp: csv_path = _hiplot_data_path(tmp.path) _write_hiplot_data(data, csv_path) html_path = _hiplot_html_path(tmp.path) _generate_hiplot_html(hiplot, csv_path, html_path) cli.out("Opening %s" % html_path, err=True) util.open_url(html_path) cli.out("To return to the prompt, press Ctrl-C", err=True) _wait_forever()
def compare_runs(get_data_cb): cli.out("Preparing data for compare", err=True) hiplot = _hiplot_include_exe() data = get_data_cb() _format_data_for_hiplot(data) html_path_env = os.getenv("HIPLOT_HTML") if html_path_env: _handle_html_env(hiplot, data, html_path_env) else: _handle_default(hiplot, data)
def _init_venv(config): cmd_args = _venv_cmd_args(config) if not cmd_args: cli.out("Skipping virtual env") return cli.out("Creating virtual environment") log.debug("venv args: %s", cmd_args) try: subprocess.check_call(cmd_args) except subprocess.CalledProcessError as e: cli.error(str(e), exit_status=e.returncode)
def _print_path(path, args): if args.full_path: path = os.path.abspath(path) elif args.sourcecode: path = _sourcecode_rel_path(path) if not path: return if args.no_format or args.full_path: cli.out(path) else: cli.out(" %s" % path)