def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): to_download = "{}{}".format(distribution, version_spec or "") logging.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) cmd = [ sys.executable, "-m", "pip", "download", "--progress-bar", "off", "--disable-pip-version-check", "--only-binary=:all:", "--no-deps", "--python-version", for_py_version, "-d", str(to_folder), to_download, ] # pip has no interface in python - must be a new sub-process env = pip_wheel_env_run(search_dirs, app_data, env) process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode != 0: kwargs = {"output": out} kwargs["stderr"] = err raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs) result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out) logging.debug("downloaded wheel %s", result.name) return result
def _run_subprocess(cls, exe, app_data): py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" with app_data.ensure_extracted(py_info_script) as py_info_script: cmd = [exe, str(py_info_script)] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = os.environ.copy() env.pop("__PYVENV_LAUNCHER__", None) logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, ) out, err = process.communicate() code = process.returncode except OSError as os_error: out, err, code = "", os_error.strerror, os_error.errno result, failure = None, None if code == 0: result = cls._from_json(out) result.executable = exe # keep original executable as this may contain initialization code else: msg = "failed to query {} with code {}{}{}".format( exe, code, " out: {!r}".format(out) if out else "", " err: {!r}".format(err) if err else "", ) failure = RuntimeError(msg) return failure, result
def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic): wheel_path = None if wheel is None else str(wheel.path) cmd = [ sys.executable, "-c", dedent( """ from virtualenv.report import setup_report, MAX_LEVEL from virtualenv.seed.wheels.periodic_update import do_update setup_report(MAX_LEVEL, show_pid=True) do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) """, ).strip().format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic), ] debug = env.get("_VIRTUALENV_PERIODIC_UPDATE_INLINE") == "1" pipe = None if debug else subprocess.PIPE kwargs = {"stdout": pipe, "stderr": pipe} if not debug and sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW process = Popen(cmd, **kwargs) logging.info( "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", distribution, "" if wheel is None else f"=={wheel.version}", for_py_version, process.pid, ) if debug: process.communicate( ) # on purpose not called to make it a background process
def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic): wheel_path = None if wheel is None else str(wheel.path) cmd = [ sys.executable, "-c", "from virtualenv.seed.wheels.periodic_update import do_update;" "do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic, ), ] debug = os.environ.get( str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") pipe = None if debug else subprocess.PIPE kwargs = {"stdout": pipe, "stderr": pipe} if not debug and sys.platform == "win32": kwargs["creationflags"] = DETACHED_PROCESS process = Popen(cmd, **kwargs) logging.info( "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", distribution, "" if wheel is None else "=={}".format(wheel.version), for_py_version, process.pid, ) if debug: process.communicate( ) # on purpose not called to make it a background process
def test_main(): process = Popen([sys.executable, "-m", "virtualenv", "--help"], universal_newlines=True, stdout=subprocess.PIPE) out, _ = process.communicate() assert not process.returncode assert out
def _generate_new_files(self): # create the pyc files, as the build image will be R/O process = Popen( [six.ensure_text(str(self._creator.exe)), "-m", "compileall", six.ensure_text(str(self._image_dir))], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) process.communicate() # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for cleanup root_py_cache = self._image_dir / "__pycache__" new_files = set() if root_py_cache.exists(): new_files.update(root_py_cache.iterdir()) new_files.add(root_py_cache) shutil.rmtree(six.ensure_text(str(root_py_cache))) core_new_files = super(SymlinkPipInstall, self)._generate_new_files() # remove files that are within the image folder deeper than one level (as these will be not linked directly) for file in core_new_files: try: rel = file.relative_to(self._image_dir) if len(rel.parts) > 1: continue except ValueError: pass new_files.add(file) return new_files
def download_wheel(packages, for_py_version, to_folder, app_data): to_download = list(p if v is None else "{}={}".format(p, v) for p, v in packages.items()) logging.debug("download wheels %s", to_download) cmd = [ sys.executable, "-m", "pip", "download", "--disable-pip-version-check", "--only-binary=:all:", "--no-deps", "--python-version", for_py_version, "-d", str(to_folder), ] cmd.extend(to_download) # pip has no interface in python - must be a new sub-process with pip_wheel_env_run("{}{}".format(*sys.version_info[0:2]), app_data) as env: process = Popen(cmd, env=env, stdout=subprocess.PIPE) process.communicate() if process.returncode != 0: raise RuntimeError("failed to download wheels")
def __call__(self, monkeypatch, tmp_path): activate_script = self._creator.bin_dir / self.activate_script test_script = self._generate_test_script(activate_script, tmp_path) monkeypatch.chdir(six.ensure_text(str(tmp_path))) monkeypatch.delenv(str("VIRTUAL_ENV"), raising=False) invoke, env = self._invoke_script + [ six.ensure_text(str(test_script)) ], self.env(tmp_path) try: process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) _raw, _ = process.communicate() raw = _raw.decode("utf-8") except subprocess.CalledProcessError as exception: assert not exception.returncode, six.ensure_text(exception.output) return out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines() self.assert_output(out, raw, tmp_path) return env, activate_script
def __call__(self, monkeypatch, tmp_path): activate_script = self._creator.bin_dir / self.activate_script # check line endings are correct type script_content = activate_script.read_bytes() for line in script_content.split(b"\n")[:-1]: cr = b"\r" if sys.version_info.major == 2 else 13 if self.unix_line_ending: assert line == b"" or line[-1] != cr, script_content.decode("utf-8") else: assert line[-1] == cr, script_content.decode("utf-8") test_script = self._generate_test_script(activate_script, tmp_path) monkeypatch.chdir(ensure_text(str(tmp_path))) monkeypatch.delenv(str("VIRTUAL_ENV"), raising=False) invoke, env = self._invoke_script + [ensure_text(str(test_script))], self.env(tmp_path) try: process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) _raw, _ = process.communicate() raw = _raw.decode("utf-8") except subprocess.CalledProcessError as exception: output = ensure_text((exception.output + exception.stderr) if six.PY3 else exception.output) assert not exception.returncode, output return out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines() self.assert_output(out, raw, tmp_path) return env, activate_script
def _run_subprocess(cls, exe): resolved_path = Path(os.path.abspath(__file__)).parent / "py_info.py" with ensure_file_on_disk(resolved_path) as resolved_path: cmd = [exe, "-s", str(resolved_path)] logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: process = Popen(cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = process.communicate() code = process.returncode except OSError as os_error: out, err, code = "", os_error.strerror, os_error.errno result, failure = None, None if code == 0: result = cls._from_json(out) result.executable = exe # keep original executable as this may contain initialization code else: msg = "failed to query {} with code {}{}{}".format( exe, code, " out: {!r}".format(out) if out else "", " err: {!r}".format(err) if err else "") failure = RuntimeError(msg) return failure, result
def _run_subprocess(cls, exe, app_data, env): py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" # Cookies allow to split the serialized stdout output generated by the script collecting the info from the output # generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor # to the child and output to it. But AFAIK all of them are either not cross-platform or too big to implement and are # not in the stdlib. So the easiest and the shortest way I could mind is just using the cookies. # We generate pseudorandom cookies because it easy to implement and avoids breakage from outputting modules source # code, i.e. by debug output libraries. We reverse the cookies to avoid breakages resulting from variable values # appearing in debug output. start_cookie = gen_cookie() end_cookie = gen_cookie() with app_data.ensure_extracted(py_info_script) as py_info_script: cmd = [exe, str(py_info_script), start_cookie, end_cookie] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, ) out, err = process.communicate() code = process.returncode except OSError as os_error: out, err, code = "", os_error.strerror, os_error.errno result, failure = None, None if code == 0: out_starts = out.find(start_cookie[::-1]) if out_starts > -1: pre_cookie = out[:out_starts] if pre_cookie: sys.stdout.write(pre_cookie) out = out[out_starts + COOKIE_LENGTH :] out_ends = out.find(end_cookie[::-1]) if out_ends > -1: post_cookie = out[out_ends + COOKIE_LENGTH :] if post_cookie: sys.stdout.write(post_cookie) out = out[:out_ends] result = cls._from_json(out) result.executable = exe # keep original executable as this may contain initialization code else: msg = f"{exe} with code {code}{f' out: {out!r}' if out else ''}{f' err: {err!r}' if err else ''}" failure = RuntimeError(f"failed to query {msg}") return failure, result
def _execute(cmd, env): logging.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() if process.returncode != 0: raise RuntimeError("failed seed with code {}".format( process.returncode)) return process
def __call__(self, monkeypatch, tmp_path): env, activate_script = super(RaiseOnNonSourceCall, self).__call__(monkeypatch, tmp_path) process = Popen( self.non_source_activate(activate_script), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) out, _err = process.communicate() err = _err.decode("utf-8") assert process.returncode assert self.non_source_fail_message in err
def venv(tmp_path_factory, session_app_data): if CURRENT.is_venv: return sys.executable elif CURRENT.version_info.major == 3: root_python = root(tmp_path_factory, session_app_data) dest = tmp_path_factory.mktemp("venv") process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) process.communicate() # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic exe_path = CURRENT.discover_exe(prefix=str(dest)).original_executable return exe_path
def run(self, creator): if not self.enabled: return with self.get_pip_install_cmd( creator.exe, creator.interpreter.version_release_str) as cmd: with pip_wheel_env_run( creator.interpreter.version_release_str) as env: logging.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() if process.returncode != 0: raise RuntimeError("failed seed with code {}".format( process.returncode))
def get_version(self, raise_on_fail): if self._version is None: # locally we disable, so that contributors don't need to have everything setup try: process = Popen( self._version_cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, err = process.communicate() result = out if out else err self._version = result return result except Exception as exception: self._version = exception if raise_on_fail: raise return RuntimeError("{} is not available due {}".format(self, exception)) return self._version
def test_can_build_c_extensions(creator, tmp_path, coverage_env): session = cli_run( ["--creator", creator, "--seed", "app-data", str(tmp_path), "-vvv"]) coverage_env() cmd = [ str(session.creator.script("pip")), "install", "--no-index", "--no-deps", "--disable-pip-version-check", "-vvv", str(Path(__file__).parent.resolve() / "greet"), ] process = Popen(cmd) process.communicate() assert process.returncode == 0 process = Popen( [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], universal_newlines=True, stdout=subprocess.PIPE, ) out, _ = process.communicate() assert process.returncode == 0 assert out == "Hello World!\n"
def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): to_download = "{}{}".format(distribution, version_spec or "") logging.debug("download wheel %s", to_download) cmd = [ sys.executable, "-m", "pip", "download", "--disable-pip-version-check", "--only-binary=:all:", "--no-deps", "--python-version", for_py_version, "-d", str(to_folder), to_download, ] # pip has no interface in python - must be a new sub-process env = pip_wheel_env_run(search_dirs, app_data) process = Popen( cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) out, err = process.communicate() if process.returncode != 0: kwargs = {"output": out} if six.PY2: kwargs["output"] += err else: kwargs["stderr"] = err raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs) for line in out.splitlines(): line = line.lstrip() for marker in ("Saved ", "File was already downloaded "): if line.startswith(marker): return Wheel(Path(line[len(marker):]).absolute()) # if for some reason the output does not match fallback to latest version with that spec return find_compatible_in_house(distribution, version_spec, for_py_version, to_folder)
def old_virtualenv(tmp_path_factory, session_app_data): if CURRENT.is_old_virtualenv: return CURRENT.executable else: env_for_old_virtualenv = tmp_path_factory.mktemp( "env-for-old-virtualenv") result = cli_run([ "--no-download", "--activators", "", str(env_for_old_virtualenv), "--no-periodic-update" ]) # noinspection PyBroadException try: process = Popen( [ str(result.creator.script("pip")), "install", "--no-index", "--disable-pip-version-check", str( Path(__file__).resolve().parent / "virtualenv-16.7.9-py2.py3-none-any.whl"), "-v", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) _, __ = process.communicate() assert not process.returncode except Exception: return RuntimeError("failed to install old virtualenv") # noinspection PyBroadException try: old_virtualenv_at = tmp_path_factory.mktemp("old-virtualenv") cmd = [ str(result.creator.script("virtualenv")), str(old_virtualenv_at), "--no-pip", "--no-setuptools", "--no-wheel", ] process = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, __ = process.communicate() assert not process.returncode if result.creator.interpreter.implementation == "PyPy" and IS_WIN: # old virtualenv creates pypy paths wrong on windows, so have to hardcode it return str(old_virtualenv_at / "bin" / "pypy.exe") exe_path = CURRENT.discover_exe( session_app_data, prefix=str(old_virtualenv_at)).original_executable return exe_path except Exception as exception: return RuntimeError( "failed to create old virtualenv {}".format(exception))
def old_virtualenv(tmp_path_factory): if CURRENT.is_old_virtualenv: return CURRENT.executable else: env_for_old_virtualenv = tmp_path_factory.mktemp( "env-for-old-virtualenv") result = cli_run( ["--no-download", "--activators", "", str(env_for_old_virtualenv)]) # noinspection PyBroadException try: process = Popen( [ str(result.creator.script("pip")), "install", "--no-index", "--disable-pip-version-check", str( Path(__file__).resolve().parent / "virtualenv-16.7.9-py2.py3-none-any.whl"), "-v", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) _, __ = process.communicate() assert not process.returncode except Exception: return RuntimeError("failed to install old virtualenv") # noinspection PyBroadException try: old_virtualenv_at = tmp_path_factory.mktemp("old-virtualenv") cmd = [ str(result.creator.script("virtualenv")), str(old_virtualenv_at), "--no-pip", "--no-setuptools", "--no-wheel", ] process = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, __ = process.communicate() assert not process.returncode exe_path = CURRENT.discover_exe( prefix=str(old_virtualenv_at)).original_executable return exe_path except Exception: return RuntimeError("failed to create old virtualenv")
def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies): current = PythonInfo.current_system() bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ ensure_text( str(tmp_path / "en v") ), # space in the name to ensure generated scripts work when path has space "--seeder", "app-data", "--extra-search-dir", ensure_text(str(BUNDLE_FOLDER)), "--download", "--pip", bundle_ver["pip"].split("-")[1], "--setuptools", bundle_ver["setuptools"].split("-")[1], "--clear-app-data", "--creator", current_fastest, "-vv", ] if not copies: create_cmd.append("--symlink-app-data") result = cli_run(create_cmd) coverage_env() assert result # uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env site_package = result.creator.purelib pip = site_package / "pip" setuptools = site_package / "setuptools" files_post_first_create = list(site_package.iterdir()) assert pip in files_post_first_create assert setuptools in files_post_first_create for pip_exe in [ result.creator.script_dir / "pip{}{}".format(suffix, result.creator.exe.suffix) for suffix in ( "", "{}".format(current.version_info.major), "{}.{}".format(current.version_info.major, current.version_info.minor), "-{}.{}".format(current.version_info.major, current.version_info.minor), ) ]: assert pip_exe.exists() process = Popen([ ensure_text(str(pip_exe)), "--version", "--disable-pip-version-check" ]) _, __ = process.communicate() assert not process.returncode remove_cmd = [ str(result.creator.script("pip")), "--verbose", "--disable-pip-version-check", "uninstall", "-y", "setuptools", ] process = Popen(remove_cmd) _, __ = process.communicate() assert not process.returncode assert site_package.exists() files_post_first_uninstall = list(site_package.iterdir()) assert pip in files_post_first_uninstall assert setuptools not in files_post_first_uninstall # check we can run it again and will work - checks both overwrite and reuse cache result = cli_run(create_cmd) coverage_env() assert result files_post_second_create = list(site_package.iterdir()) assert files_post_first_create == files_post_second_create # Windows does not allow removing a executable while running it, so when uninstalling pip we need to do it via # python -m pip remove_cmd = [str(result.creator.exe), "-m", "pip"] + remove_cmd[1:] process = Popen(remove_cmd + ["pip", "wheel"]) _, __ = process.communicate() assert not process.returncode # pip is greedy here, removing all packages removes the site-package too if site_package.exists(): purelib = result.creator.purelib patch_files = { purelib / "{}.{}".format("_virtualenv", i) for i in ("py", "pyc", "pth") } patch_files.add(purelib / "__pycache__") post_run = set(site_package.iterdir()) - patch_files assert not post_run, "\n".join(str(i) for i in post_run) if sys.version_info[0:2] == (3, 4) and os.environ.get( str("PIP_REQ_TRACKER")): os.environ.pop(str("PIP_REQ_TRACKER"))
def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env, current_fastest): current = PythonInfo.current_system() bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ six.ensure_text(str(tmp_path / "env")), "--seeder", "app-data", "--extra-search-dir", six.ensure_text(str(BUNDLE_FOLDER)), "--download", "--pip", bundle_ver["pip"].split("-")[1], "--setuptools", bundle_ver["setuptools"].split("-")[1], "--clear-app-data", "--creator", current_fastest, "-vv", ] result = run_via_cli(create_cmd) coverage_env() assert result # uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env site_package = result.creator.purelib pip = site_package / "pip" setuptools = site_package / "setuptools" files_post_first_create = list(site_package.iterdir()) assert pip in files_post_first_create assert setuptools in files_post_first_create for pip_exe in [ result.creator.script_dir / "pip{}{}".format(suffix, result.creator.exe.suffix) for suffix in ( "", "{}".format(current.version_info.major), "-{}.{}".format(current.version_info.major, current.version_info.minor), ) ]: assert pip_exe.exists() process = Popen([ six.ensure_text(str(pip_exe)), "--version", "--disable-pip-version-check" ]) _, __ = process.communicate() assert not process.returncode remove_cmd = [ str(result.creator.exe), "-m", "pip", "--verbose", "--disable-pip-version-check", "uninstall", "-y", "setuptools", ] process = Popen(remove_cmd) _, __ = process.communicate() assert not process.returncode assert site_package.exists() files_post_first_uninstall = list(site_package.iterdir()) assert pip in files_post_first_uninstall assert setuptools not in files_post_first_uninstall # check we can run it again and will work - checks both overwrite and reuse cache result = run_via_cli(create_cmd) coverage_env() assert result files_post_second_create = list(site_package.iterdir()) assert files_post_first_create == files_post_second_create process = Popen(remove_cmd + ["pip", "wheel"]) _, __ = process.communicate() assert not process.returncode # pip is greedy here, removing all packages removes the site-package too if site_package.exists(): post_run = list(site_package.iterdir()) assert not post_run, "\n".join(str(i) for i in post_run) if sys.version_info[0:2] == (3, 4) and os.environ.get( str("PIP_REQ_TRACKER")): os.environ.pop(str("PIP_REQ_TRACKER"))