def _init_cmd_env(opdef): env = {} env.update(util.safe_osenv()) env["GUILD_PLUGINS"] = _op_plugins(opdef) env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel()) env["PYTHONPATH"] = _python_path(opdef) return env
def _init_cmd_env(opdef, restart, gpus): env = util.safe_osenv() env.update({ name: str(val) for name, val in opdef.env.items() }) env["GUILD_HOME"] = configlib.guild_home() env["GUILD_OP"] = opdef.fullname env["GUILD_PLUGINS"] = _op_plugins(opdef) env["LOG_LEVEL"] = _log_level() env["PYTHONPATH"] = _python_path(opdef, restart) # SCRIPT_DIR is set by op_main at sys.path[0] - use empty string # here to include run dir first in sys.path env["SCRIPT_DIR"] = "" # CMD_DIR is where the operation cmd was run env["CMD_DIR"] = os.getcwd() env["MODEL_DIR"] = opdef.guildfile.dir env["MODEL_PATH"] = os.path.pathsep.join(_model_paths(opdef)) if opdef.flags_dest: env["FLAGS_DEST"] = opdef.flags_dest if opdef.set_trace: env["SET_TRACE"] = "1" if opdef.handle_keyboard_interrupt: env["HANDLE_KEYBOARD_INTERRUPT"] = "1" util.apply_env(env, os.environ, ["PROFILE"]) if gpus is not None: log.info( "Limiting available GPUs (CUDA_VISIBLE_DEVICES) to: %s", gpus or "<none>") env["CUDA_VISIBLE_DEVICES"] = gpus return env
def _op_proc_env_system(op): env = util.safe_osenv() env["GUILD_HOME"] = config.guild_home() env["LOG_LEVEL"] = _log_level() env["PYTHONPATH"] = _python_path(op) env["CMD_DIR"] = os.getcwd() return env
def create_package(package_file, dist_dir=None, upload_repo=False, sign=False, identity=None, user=None, password=None, comment=None): # Use a separate OS process as setup assumes it's running as a # command line op. We make sure to import package_main lazily here # because it incurs various runtime deps that we don't want to # load actively. cmd = [sys.executable, "-um", "guild.package_main"] env = {} env.update(util.safe_osenv()) env.update({ "PYTHONPATH": os.path.pathsep.join(sys.path), "PACKAGE_FILE": package_file, "DIST_DIR": dist_dir or "", "UPLOAD_REPO": upload_repo or "", "SIGN": "1" if sign else "", "IDENTITY": identity or "", "USER": user or "", "PASSWORD": password or "", "COMMENT": comment or "", }) p = subprocess.Popen(cmd, env=env) exit_code = p.wait() if exit_code != 0: raise SystemExit(exit_code)
def create_package( package_file, clean=False, dist_dir=None, upload_repo=False, sign=False, identity=None, user=None, password=None, skip_existing=False, comment=None, capture_output=False, ): # Use a separate OS process as setup assumes it's running as a # command line op. cmd = [sys.executable, "-um", "guild.package_main"] env = {} env.update(util.safe_osenv()) env.update({ "COMMENT": comment or "", "DEBUG": log.getEffectiveLevel() <= logging.DEBUG and "1" or "", "DIST_DIR": dist_dir or "", "IDENTITY": identity or "", "LOG_LEVEL": str(log.getEffectiveLevel()), "PACKAGE_FILE": package_file, "PASSWORD": password or "", "PYTHONPATH": _python_path(), "SIGN": "1" if sign else "", "SKIP_EXISTING": skip_existing and "1" or "", "UPLOAD_REPO": upload_repo or "", "USER": user or "", }) if clean: env["CLEAN_SETUP"] = "1" _apply_twine_env_creds(env) cwd = os.path.dirname(package_file) if capture_output: stdout = subprocess.PIPE stderr = subprocess.STDOUT else: stdout = None stderr = None log.debug("package cmd: %s", cmd) log.debug("package env: %s", env) log.debug("package cwd: %s", cwd) p = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=stdout, stderr=stderr) out, _ = p.communicate() if p.returncode != 0: if capture_output: raise SystemExit(p.returncode, out.decode()) else: raise SystemExit(p.returncode) return out
def _print_tensorflow_info(check): # Run externally to avoid tf logging to our stderr cmd = [sys.executable, "-um", "guild.commands.tensorflow_check_main"] env = util.safe_osenv() env["PYTHONPATH"] = os.path.pathsep.join(sys.path) if check.args.verbose: stderr = None else: stderr = open(os.devnull, "w") p = subprocess.Popen(cmd, stderr=stderr, env=env) exit_status = p.wait() if exit_status != 0: check.error()
def _op_proc_system_env(): return util.safe_osenv()