def _init_cmd_env(opdef):
env = {}
env.update(util.safe_osenv())
env["GUILD_PLUGINS"] = _op_plugins(opdef)
env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel())
env["PYTHONPATH"] = _python_path(opdef)
return env
def _init_cmd_env(opdef, restart, gpus):
env = util.safe_osenv()
env.update({
name: str(val)
for name, val in opdef.env.items()
})
env["GUILD_HOME"] = configlib.guild_home()
env["GUILD_OP"] = opdef.fullname
env["GUILD_PLUGINS"] = _op_plugins(opdef)
env["LOG_LEVEL"] = _log_level()
env["PYTHONPATH"] = _python_path(opdef, restart)
# SCRIPT_DIR is set by op_main at sys.path[0] - use empty string
# here to include run dir first in sys.path
env["SCRIPT_DIR"] = ""
# CMD_DIR is where the operation cmd was run
env["CMD_DIR"] = os.getcwd()
env["MODEL_DIR"] = opdef.guildfile.dir
env["MODEL_PATH"] = os.path.pathsep.join(_model_paths(opdef))
if opdef.flags_dest:
env["FLAGS_DEST"] = opdef.flags_dest
if opdef.set_trace:
env["SET_TRACE"] = "1"
if opdef.handle_keyboard_interrupt:
env["HANDLE_KEYBOARD_INTERRUPT"] = "1"
util.apply_env(env, os.environ, ["PROFILE"])
if gpus is not None:
log.info(
"Limiting available GPUs (CUDA_VISIBLE_DEVICES) to: %s",
gpus or "<none>")
env["CUDA_VISIBLE_DEVICES"] = gpus
return env
def _op_proc_env_system(op):
env = util.safe_osenv()
env["GUILD_HOME"] = config.guild_home()
env["LOG_LEVEL"] = _log_level()
env["PYTHONPATH"] = _python_path(op)
env["CMD_DIR"] = os.getcwd()
return env
def create_package(package_file,
dist_dir=None,
upload_repo=False,
sign=False,
identity=None,
user=None,
password=None,
comment=None):
# Use a separate OS process as setup assumes it's running as a
# command line op. We make sure to import package_main lazily here
# because it incurs various runtime deps that we don't want to
# load actively.
cmd = [sys.executable, "-um", "guild.package_main"]
env = {}
env.update(util.safe_osenv())
env.update({
"PYTHONPATH": os.path.pathsep.join(sys.path),
"PACKAGE_FILE": package_file,
"DIST_DIR": dist_dir or "",
"UPLOAD_REPO": upload_repo or "",
"SIGN": "1" if sign else "",
"IDENTITY": identity or "",
"USER": user or "",
"PASSWORD": password or "",
"COMMENT": comment or "",
})
p = subprocess.Popen(cmd, env=env)
exit_code = p.wait()
if exit_code != 0:
raise SystemExit(exit_code)
def create_package(
package_file,
clean=False,
dist_dir=None,
upload_repo=False,
sign=False,
identity=None,
user=None,
password=None,
skip_existing=False,
comment=None,
capture_output=False,
):
# Use a separate OS process as setup assumes it's running as a
# command line op.
cmd = [sys.executable, "-um", "guild.package_main"]
env = {}
env.update(util.safe_osenv())
env.update({
"COMMENT": comment or "",
"DEBUG": log.getEffectiveLevel() <= logging.DEBUG and "1" or "",
"DIST_DIR": dist_dir or "",
"IDENTITY": identity or "",
"LOG_LEVEL": str(log.getEffectiveLevel()),
"PACKAGE_FILE": package_file,
"PASSWORD": password or "",
"PYTHONPATH": _python_path(),
"SIGN": "1" if sign else "",
"SKIP_EXISTING": skip_existing and "1" or "",
"UPLOAD_REPO": upload_repo or "",
"USER": user or "",
})
if clean:
env["CLEAN_SETUP"] = "1"
_apply_twine_env_creds(env)
cwd = os.path.dirname(package_file)
if capture_output:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
else:
stdout = None
stderr = None
log.debug("package cmd: %s", cmd)
log.debug("package env: %s", env)
log.debug("package cwd: %s", cwd)
p = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=stdout, stderr=stderr)
out, _ = p.communicate()
if p.returncode != 0:
if capture_output:
raise SystemExit(p.returncode, out.decode())
else:
raise SystemExit(p.returncode)
return out
def _print_tensorflow_info(check):
# Run externally to avoid tf logging to our stderr
cmd = [sys.executable, "-um", "guild.commands.tensorflow_check_main"]
env = util.safe_osenv()
env["PYTHONPATH"] = os.path.pathsep.join(sys.path)
if check.args.verbose:
stderr = None
else:
stderr = open(os.devnull, "w")
p = subprocess.Popen(cmd, stderr=stderr, env=env)
exit_status = p.wait()
if exit_status != 0:
check.error()
def _op_proc_system_env():
return util.safe_osenv()
