def _print_space(): cli.out("disk_space:") _print_disk_usage("guild_home", config.guild_home()) _print_disk_usage("runs", var.runs_dir()) _print_disk_usage("deleted_runs", var.runs_dir(deleted=True)) _print_disk_usage("remote_state", var.remote_dir()) _print_disk_usage("cache", var.cache_dir())
def _init_cmd_env(opdef, restart, gpus): env = util.safe_osenv() env.update({ name: str(val) for name, val in opdef.env.items() }) env["GUILD_HOME"] = configlib.guild_home() env["GUILD_OP"] = opdef.fullname env["GUILD_PLUGINS"] = _op_plugins(opdef) env["LOG_LEVEL"] = _log_level() env["PYTHONPATH"] = _python_path(opdef, restart) # SCRIPT_DIR is set by op_main at sys.path[0] - use empty string # here to include run dir first in sys.path env["SCRIPT_DIR"] = "" # CMD_DIR is where the operation cmd was run env["CMD_DIR"] = os.getcwd() env["MODEL_DIR"] = opdef.guildfile.dir env["MODEL_PATH"] = os.path.pathsep.join(_model_paths(opdef)) if opdef.flags_dest: env["FLAGS_DEST"] = opdef.flags_dest if opdef.set_trace: env["SET_TRACE"] = "1" if opdef.handle_keyboard_interrupt: env["HANDLE_KEYBOARD_INTERRUPT"] = "1" util.apply_env(env, os.environ, ["PROFILE"]) if gpus is not None: log.info( "Limiting available GPUs (CUDA_VISIBLE_DEVICES) to: %s", gpus or "<none>") env["CUDA_VISIBLE_DEVICES"] = gpus return env
def _op_proc_env_system(op): env = util.safe_osenv() env["GUILD_HOME"] = config.guild_home() env["LOG_LEVEL"] = _log_level() env["PYTHONPATH"] = _python_path(op) env["CMD_DIR"] = os.getcwd() return env
def _run( cmd, quiet=False, ignore=None, timeout=60, cut=None, guild_home=None, cwd=None, env=None, _capture=False, ): cmd = _run_shell_cmd(cmd) proc_env = dict(os.environ) if env: proc_env.update(env) proc_env["SYNC_RUN_OUTPUT"] = "1" proc_env["GUILD_HOME"] = guild_home or configlib.guild_home() p = _popen(cmd, proc_env, cwd) with _kill_after(p, timeout): out, err = p.communicate() assert err is None, err exit_code = p.returncode if _capture or not quiet or exit_code != 0: out = out.strip().decode("latin-1") if ignore: out = _strip_lines(out, ignore) if cut: out = _cut_cols(out, cut) if _capture: return out, exit_code print(out) print("<exit %i>" % exit_code)
def _op_proc_op_env(op): env = {} env.update(op.cmd_env) if op.opref: env["GUILD_OP"] = op.opref.to_opspec() env["GUILD_HOME"] = config.guild_home() env["GUILD_SOURCECODE"] = _guild_sourcecode_env(op) env["LOG_LEVEL"] = _log_level() env["PYTHONPATH"] = _python_path(op) env["CMD_DIR"] = os.getcwd() return env
def _print_disk_usage(): cli.out("disk_space:") paths = [ ("guild_home", config.guild_home()), ("runs", var.runs_dir()), ("deleted_runs", var.runs_dir(deleted=True)), ("remote_state", var.remote_dir()), ("cache", var.cache_dir()), ] formatted_disk_usage = [ _formatted_disk_usage(path) for _name, path in paths ] max_disk_usage_width = max([len(s) for s in formatted_disk_usage]) for (name, path), disk_usage in zip(paths, formatted_disk_usage): _attr( " %s" % name, _format_disk_usage_and_path(disk_usage, path, max_disk_usage_width), )
def __init__(self, cwd, guild_home=None): from guild import config guild_home = guild_home or config.guild_home() self._set_cwd = config.SetCwd(cwd or ".") self._set_guild_home = config.SetGuildHome(guild_home)
def _config_guild_home(): from guild import config return config.guild_home()
def _print_guild_info(): cli.out("guild_version: %s" % guild.version()) cli.out("guild_install_location: %s" % _guild_install_location()) cli.out("guild_home: %s" % config.guild_home()) cli.out("guild_resource_cache: %s" % _guild_resource_cache()) cli.out("installed_plugins: %s" % _format_plugins())
def _init_guild_home(args): if args.guild_home: return args.guild_home if args.no_isolate: return config.guild_home() return None
def guild_home(): return config.guild_home()
def Lock(name, timeout=-1, guild_home=None): guild_home = guild_home or config.guild_home() locks_dir = os.path.join(guild_home, "locks") util.ensure_dir(locks_dir) lock_path = os.path.join(locks_dir, name) return filelock.FileLock(lock_path, timeout)
def path(*names): names = [name for name in names if name] return os.path.join(config.guild_home(), *names)