def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env):
to_download = "{}{}".format(distribution, version_spec or "")
logging.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder)
cmd = [
sys.executable,
"-m",
"pip",
"download",
"--progress-bar",
"off",
"--disable-pip-version-check",
"--only-binary=:all:",
"--no-deps",
"--python-version",
for_py_version,
"-d",
str(to_folder),
to_download,
]
# pip has no interface in python - must be a new sub-process
env = pip_wheel_env_run(search_dirs, app_data, env)
process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
out, err = process.communicate()
if process.returncode != 0:
kwargs = {"output": out}
kwargs["stderr"] = err
raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs)
result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out)
logging.debug("downloaded wheel %s", result.name)
return result
def _run_subprocess(cls, exe, app_data):
py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py"
with app_data.ensure_extracted(py_info_script) as py_info_script:
cmd = [exe, str(py_info_script)]
# prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490
env = os.environ.copy()
env.pop("__PYVENV_LAUNCHER__", None)
logging.debug("get interpreter info via cmd: %s", LogCmd(cmd))
try:
process = Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
)
out, err = process.communicate()
code = process.returncode
except OSError as os_error:
out, err, code = "", os_error.strerror, os_error.errno
result, failure = None, None
if code == 0:
result = cls._from_json(out)
result.executable = exe # keep original executable as this may contain initialization code
else:
msg = "failed to query {} with code {}{}{}".format(
exe,
code,
" out: {!r}".format(out) if out else "",
" err: {!r}".format(err) if err else "",
)
failure = RuntimeError(msg)
return failure, result
def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data,
env, periodic):
wheel_path = None if wheel is None else str(wheel.path)
cmd = [
sys.executable,
"-c",
dedent(
"""
from virtualenv.report import setup_report, MAX_LEVEL
from virtualenv.seed.wheels.periodic_update import do_update
setup_report(MAX_LEVEL, show_pid=True)
do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})
""", ).strip().format(distribution, for_py_version, wheel_path,
str(app_data), [str(p) for p in search_dirs],
periodic),
]
debug = env.get("_VIRTUALENV_PERIODIC_UPDATE_INLINE") == "1"
pipe = None if debug else subprocess.PIPE
kwargs = {"stdout": pipe, "stderr": pipe}
if not debug and sys.platform == "win32":
kwargs["creationflags"] = CREATE_NO_WINDOW
process = Popen(cmd, **kwargs)
logging.info(
"triggered periodic upgrade of %s%s (for python %s) via background process having PID %d",
distribution,
"" if wheel is None else f"=={wheel.version}",
for_py_version,
process.pid,
)
if debug:
process.communicate(
) # on purpose not called to make it a background process
def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data,
periodic):
wheel_path = None if wheel is None else str(wheel.path)
cmd = [
sys.executable,
"-c",
"from virtualenv.seed.wheels.periodic_update import do_update;"
"do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
distribution,
for_py_version,
wheel_path,
str(app_data),
[str(p) for p in search_dirs],
periodic,
),
]
debug = os.environ.get(
str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1")
pipe = None if debug else subprocess.PIPE
kwargs = {"stdout": pipe, "stderr": pipe}
if not debug and sys.platform == "win32":
kwargs["creationflags"] = DETACHED_PROCESS
process = Popen(cmd, **kwargs)
logging.info(
"triggered periodic upgrade of %s%s (for python %s) via background process having PID %d",
distribution,
"" if wheel is None else "=={}".format(wheel.version),
for_py_version,
process.pid,
)
if debug:
process.communicate(
) # on purpose not called to make it a background process
def test_main():
process = Popen([sys.executable, "-m", "virtualenv", "--help"],
universal_newlines=True,
stdout=subprocess.PIPE)
out, _ = process.communicate()
assert not process.returncode
assert out
def _generate_new_files(self):
# create the pyc files, as the build image will be R/O
process = Popen(
[six.ensure_text(str(self._creator.exe)), "-m", "compileall", six.ensure_text(str(self._image_dir))],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
process.communicate()
# the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for cleanup
root_py_cache = self._image_dir / "__pycache__"
new_files = set()
if root_py_cache.exists():
new_files.update(root_py_cache.iterdir())
new_files.add(root_py_cache)
shutil.rmtree(six.ensure_text(str(root_py_cache)))
core_new_files = super(SymlinkPipInstall, self)._generate_new_files()
# remove files that are within the image folder deeper than one level (as these will be not linked directly)
for file in core_new_files:
try:
rel = file.relative_to(self._image_dir)
if len(rel.parts) > 1:
continue
except ValueError:
pass
new_files.add(file)
return new_files
def download_wheel(packages, for_py_version, to_folder, app_data):
to_download = list(p if v is None else "{}={}".format(p, v)
for p, v in packages.items())
logging.debug("download wheels %s", to_download)
cmd = [
sys.executable,
"-m",
"pip",
"download",
"--disable-pip-version-check",
"--only-binary=:all:",
"--no-deps",
"--python-version",
for_py_version,
"-d",
str(to_folder),
]
cmd.extend(to_download)
# pip has no interface in python - must be a new sub-process
with pip_wheel_env_run("{}{}".format(*sys.version_info[0:2]),
app_data) as env:
process = Popen(cmd, env=env, stdout=subprocess.PIPE)
process.communicate()
if process.returncode != 0:
raise RuntimeError("failed to download wheels")
def __call__(self, monkeypatch, tmp_path):
activate_script = self._creator.bin_dir / self.activate_script
test_script = self._generate_test_script(activate_script, tmp_path)
monkeypatch.chdir(six.ensure_text(str(tmp_path)))
monkeypatch.delenv(str("VIRTUAL_ENV"), raising=False)
invoke, env = self._invoke_script + [
six.ensure_text(str(test_script))
], self.env(tmp_path)
try:
process = Popen(invoke,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
_raw, _ = process.communicate()
raw = _raw.decode("utf-8")
except subprocess.CalledProcessError as exception:
assert not exception.returncode, six.ensure_text(exception.output)
return
out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw,
re.M).strip().splitlines()
self.assert_output(out, raw, tmp_path)
return env, activate_script
def __call__(self, monkeypatch, tmp_path):
activate_script = self._creator.bin_dir / self.activate_script
# check line endings are correct type
script_content = activate_script.read_bytes()
for line in script_content.split(b"\n")[:-1]:
cr = b"\r" if sys.version_info.major == 2 else 13
if self.unix_line_ending:
assert line == b"" or line[-1] != cr, script_content.decode("utf-8")
else:
assert line[-1] == cr, script_content.decode("utf-8")
test_script = self._generate_test_script(activate_script, tmp_path)
monkeypatch.chdir(ensure_text(str(tmp_path)))
monkeypatch.delenv(str("VIRTUAL_ENV"), raising=False)
invoke, env = self._invoke_script + [ensure_text(str(test_script))], self.env(tmp_path)
try:
process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
_raw, _ = process.communicate()
raw = _raw.decode("utf-8")
except subprocess.CalledProcessError as exception:
output = ensure_text((exception.output + exception.stderr) if six.PY3 else exception.output)
assert not exception.returncode, output
return
out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines()
self.assert_output(out, raw, tmp_path)
return env, activate_script
def _run_subprocess(cls, exe):
resolved_path = Path(os.path.abspath(__file__)).parent / "py_info.py"
with ensure_file_on_disk(resolved_path) as resolved_path:
cmd = [exe, "-s", str(resolved_path)]
logging.debug("get interpreter info via cmd: %s", LogCmd(cmd))
try:
process = Popen(cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = process.communicate()
code = process.returncode
except OSError as os_error:
out, err, code = "", os_error.strerror, os_error.errno
result, failure = None, None
if code == 0:
result = cls._from_json(out)
result.executable = exe # keep original executable as this may contain initialization code
else:
msg = "failed to query {} with code {}{}{}".format(
exe, code, " out: {!r}".format(out) if out else "",
" err: {!r}".format(err) if err else "")
failure = RuntimeError(msg)
return failure, result
def _run_subprocess(cls, exe, app_data, env):
py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py"
# Cookies allow to split the serialized stdout output generated by the script collecting the info from the output
# generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor
# to the child and output to it. But AFAIK all of them are either not cross-platform or too big to implement and are
# not in the stdlib. So the easiest and the shortest way I could mind is just using the cookies.
# We generate pseudorandom cookies because it easy to implement and avoids breakage from outputting modules source
# code, i.e. by debug output libraries. We reverse the cookies to avoid breakages resulting from variable values
# appearing in debug output.
start_cookie = gen_cookie()
end_cookie = gen_cookie()
with app_data.ensure_extracted(py_info_script) as py_info_script:
cmd = [exe, str(py_info_script), start_cookie, end_cookie]
# prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490
env = env.copy()
env.pop("__PYVENV_LAUNCHER__", None)
logging.debug("get interpreter info via cmd: %s", LogCmd(cmd))
try:
process = Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
)
out, err = process.communicate()
code = process.returncode
except OSError as os_error:
out, err, code = "", os_error.strerror, os_error.errno
result, failure = None, None
if code == 0:
out_starts = out.find(start_cookie[::-1])
if out_starts > -1:
pre_cookie = out[:out_starts]
if pre_cookie:
sys.stdout.write(pre_cookie)
out = out[out_starts + COOKIE_LENGTH :]
out_ends = out.find(end_cookie[::-1])
if out_ends > -1:
post_cookie = out[out_ends + COOKIE_LENGTH :]
if post_cookie:
sys.stdout.write(post_cookie)
out = out[:out_ends]
result = cls._from_json(out)
result.executable = exe # keep original executable as this may contain initialization code
else:
msg = f"{exe} with code {code}{f' out: {out!r}' if out else ''}{f' err: {err!r}' if err else ''}"
failure = RuntimeError(f"failed to query {msg}")
return failure, result
def _execute(cmd, env):
logging.debug("pip seed by running: %s", LogCmd(cmd, env))
process = Popen(cmd, env=env)
process.communicate()
if process.returncode != 0:
raise RuntimeError("failed seed with code {}".format(
process.returncode))
return process
def __call__(self, monkeypatch, tmp_path):
env, activate_script = super(RaiseOnNonSourceCall, self).__call__(monkeypatch, tmp_path)
process = Popen(
self.non_source_activate(activate_script), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env,
)
out, _err = process.communicate()
err = _err.decode("utf-8")
assert process.returncode
assert self.non_source_fail_message in err
def venv(tmp_path_factory, session_app_data):
if CURRENT.is_venv:
return sys.executable
elif CURRENT.version_info.major == 3:
root_python = root(tmp_path_factory, session_app_data)
dest = tmp_path_factory.mktemp("venv")
process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)])
process.communicate()
# sadly creating a virtual environment does not tell us where the executable lives in general case
# so discover using some heuristic
exe_path = CURRENT.discover_exe(prefix=str(dest)).original_executable
return exe_path
def run(self, creator):
if not self.enabled:
return
with self.get_pip_install_cmd(
creator.exe, creator.interpreter.version_release_str) as cmd:
with pip_wheel_env_run(
creator.interpreter.version_release_str) as env:
logging.debug("pip seed by running: %s", LogCmd(cmd, env))
process = Popen(cmd, env=env)
process.communicate()
if process.returncode != 0:
raise RuntimeError("failed seed with code {}".format(
process.returncode))
def get_version(self, raise_on_fail):
if self._version is None:
# locally we disable, so that contributors don't need to have everything setup
try:
process = Popen(
self._version_cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
out, err = process.communicate()
result = out if out else err
self._version = result
return result
except Exception as exception:
self._version = exception
if raise_on_fail:
raise
return RuntimeError("{} is not available due {}".format(self, exception))
return self._version
def test_can_build_c_extensions(creator, tmp_path, coverage_env):
session = cli_run(
["--creator", creator, "--seed", "app-data",
str(tmp_path), "-vvv"])
coverage_env()
cmd = [
str(session.creator.script("pip")),
"install",
"--no-index",
"--no-deps",
"--disable-pip-version-check",
"-vvv",
str(Path(__file__).parent.resolve() / "greet"),
]
process = Popen(cmd)
process.communicate()
assert process.returncode == 0
process = Popen(
[str(session.creator.exe), "-c", "import greet; greet.greet('World')"],
universal_newlines=True,
stdout=subprocess.PIPE,
)
out, _ = process.communicate()
assert process.returncode == 0
assert out == "Hello World!\n"
def download_wheel(distribution, version_spec, for_py_version, search_dirs,
app_data, to_folder):
to_download = "{}{}".format(distribution, version_spec or "")
logging.debug("download wheel %s", to_download)
cmd = [
sys.executable,
"-m",
"pip",
"download",
"--disable-pip-version-check",
"--only-binary=:all:",
"--no-deps",
"--python-version",
for_py_version,
"-d",
str(to_folder),
to_download,
]
# pip has no interface in python - must be a new sub-process
env = pip_wheel_env_run(search_dirs, app_data)
process = Popen(
cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
out, err = process.communicate()
if process.returncode != 0:
kwargs = {"output": out}
if six.PY2:
kwargs["output"] += err
else:
kwargs["stderr"] = err
raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs)
for line in out.splitlines():
line = line.lstrip()
for marker in ("Saved ", "File was already downloaded "):
if line.startswith(marker):
return Wheel(Path(line[len(marker):]).absolute())
# if for some reason the output does not match fallback to latest version with that spec
return find_compatible_in_house(distribution, version_spec, for_py_version,
to_folder)
def old_virtualenv(tmp_path_factory, session_app_data):
if CURRENT.is_old_virtualenv:
return CURRENT.executable
else:
env_for_old_virtualenv = tmp_path_factory.mktemp(
"env-for-old-virtualenv")
result = cli_run([
"--no-download", "--activators", "",
str(env_for_old_virtualenv), "--no-periodic-update"
])
# noinspection PyBroadException
try:
process = Popen(
[
str(result.creator.script("pip")),
"install",
"--no-index",
"--disable-pip-version-check",
str(
Path(__file__).resolve().parent /
"virtualenv-16.7.9-py2.py3-none-any.whl"),
"-v",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_, __ = process.communicate()
assert not process.returncode
except Exception:
return RuntimeError("failed to install old virtualenv")
# noinspection PyBroadException
try:
old_virtualenv_at = tmp_path_factory.mktemp("old-virtualenv")
cmd = [
str(result.creator.script("virtualenv")),
str(old_virtualenv_at),
"--no-pip",
"--no-setuptools",
"--no-wheel",
]
process = Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, __ = process.communicate()
assert not process.returncode
if result.creator.interpreter.implementation == "PyPy" and IS_WIN:
# old virtualenv creates pypy paths wrong on windows, so have to hardcode it
return str(old_virtualenv_at / "bin" / "pypy.exe")
exe_path = CURRENT.discover_exe(
session_app_data,
prefix=str(old_virtualenv_at)).original_executable
return exe_path
except Exception as exception:
return RuntimeError(
"failed to create old virtualenv {}".format(exception))
def old_virtualenv(tmp_path_factory):
if CURRENT.is_old_virtualenv:
return CURRENT.executable
else:
env_for_old_virtualenv = tmp_path_factory.mktemp(
"env-for-old-virtualenv")
result = cli_run(
["--no-download", "--activators", "",
str(env_for_old_virtualenv)])
# noinspection PyBroadException
try:
process = Popen(
[
str(result.creator.script("pip")),
"install",
"--no-index",
"--disable-pip-version-check",
str(
Path(__file__).resolve().parent /
"virtualenv-16.7.9-py2.py3-none-any.whl"),
"-v",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_, __ = process.communicate()
assert not process.returncode
except Exception:
return RuntimeError("failed to install old virtualenv")
# noinspection PyBroadException
try:
old_virtualenv_at = tmp_path_factory.mktemp("old-virtualenv")
cmd = [
str(result.creator.script("virtualenv")),
str(old_virtualenv_at),
"--no-pip",
"--no-setuptools",
"--no-wheel",
]
process = Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, __ = process.communicate()
assert not process.returncode
exe_path = CURRENT.discover_exe(
prefix=str(old_virtualenv_at)).original_executable
return exe_path
except Exception:
return RuntimeError("failed to create old virtualenv")
def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest,
copies):
current = PythonInfo.current_system()
bundle_ver = BUNDLE_SUPPORT[current.version_release_str]
create_cmd = [
ensure_text(
str(tmp_path / "en v")
), # space in the name to ensure generated scripts work when path has space
"--seeder",
"app-data",
"--extra-search-dir",
ensure_text(str(BUNDLE_FOLDER)),
"--download",
"--pip",
bundle_ver["pip"].split("-")[1],
"--setuptools",
bundle_ver["setuptools"].split("-")[1],
"--clear-app-data",
"--creator",
current_fastest,
"-vv",
]
if not copies:
create_cmd.append("--symlink-app-data")
result = cli_run(create_cmd)
coverage_env()
assert result
# uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env
site_package = result.creator.purelib
pip = site_package / "pip"
setuptools = site_package / "setuptools"
files_post_first_create = list(site_package.iterdir())
assert pip in files_post_first_create
assert setuptools in files_post_first_create
for pip_exe in [
result.creator.script_dir /
"pip{}{}".format(suffix, result.creator.exe.suffix) for suffix in (
"",
"{}".format(current.version_info.major),
"{}.{}".format(current.version_info.major,
current.version_info.minor),
"-{}.{}".format(current.version_info.major,
current.version_info.minor),
)
]:
assert pip_exe.exists()
process = Popen([
ensure_text(str(pip_exe)), "--version",
"--disable-pip-version-check"
])
_, __ = process.communicate()
assert not process.returncode
remove_cmd = [
str(result.creator.script("pip")),
"--verbose",
"--disable-pip-version-check",
"uninstall",
"-y",
"setuptools",
]
process = Popen(remove_cmd)
_, __ = process.communicate()
assert not process.returncode
assert site_package.exists()
files_post_first_uninstall = list(site_package.iterdir())
assert pip in files_post_first_uninstall
assert setuptools not in files_post_first_uninstall
# check we can run it again and will work - checks both overwrite and reuse cache
result = cli_run(create_cmd)
coverage_env()
assert result
files_post_second_create = list(site_package.iterdir())
assert files_post_first_create == files_post_second_create
# Windows does not allow removing a executable while running it, so when uninstalling pip we need to do it via
# python -m pip
remove_cmd = [str(result.creator.exe), "-m", "pip"] + remove_cmd[1:]
process = Popen(remove_cmd + ["pip", "wheel"])
_, __ = process.communicate()
assert not process.returncode
# pip is greedy here, removing all packages removes the site-package too
if site_package.exists():
purelib = result.creator.purelib
patch_files = {
purelib / "{}.{}".format("_virtualenv", i)
for i in ("py", "pyc", "pth")
}
patch_files.add(purelib / "__pycache__")
post_run = set(site_package.iterdir()) - patch_files
assert not post_run, "\n".join(str(i) for i in post_run)
if sys.version_info[0:2] == (3, 4) and os.environ.get(
str("PIP_REQ_TRACKER")):
os.environ.pop(str("PIP_REQ_TRACKER"))
def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env,
current_fastest):
current = PythonInfo.current_system()
bundle_ver = BUNDLE_SUPPORT[current.version_release_str]
create_cmd = [
six.ensure_text(str(tmp_path / "env")),
"--seeder",
"app-data",
"--extra-search-dir",
six.ensure_text(str(BUNDLE_FOLDER)),
"--download",
"--pip",
bundle_ver["pip"].split("-")[1],
"--setuptools",
bundle_ver["setuptools"].split("-")[1],
"--clear-app-data",
"--creator",
current_fastest,
"-vv",
]
result = run_via_cli(create_cmd)
coverage_env()
assert result
# uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env
site_package = result.creator.purelib
pip = site_package / "pip"
setuptools = site_package / "setuptools"
files_post_first_create = list(site_package.iterdir())
assert pip in files_post_first_create
assert setuptools in files_post_first_create
for pip_exe in [
result.creator.script_dir /
"pip{}{}".format(suffix, result.creator.exe.suffix) for suffix in (
"",
"{}".format(current.version_info.major),
"-{}.{}".format(current.version_info.major,
current.version_info.minor),
)
]:
assert pip_exe.exists()
process = Popen([
six.ensure_text(str(pip_exe)), "--version",
"--disable-pip-version-check"
])
_, __ = process.communicate()
assert not process.returncode
remove_cmd = [
str(result.creator.exe),
"-m",
"pip",
"--verbose",
"--disable-pip-version-check",
"uninstall",
"-y",
"setuptools",
]
process = Popen(remove_cmd)
_, __ = process.communicate()
assert not process.returncode
assert site_package.exists()
files_post_first_uninstall = list(site_package.iterdir())
assert pip in files_post_first_uninstall
assert setuptools not in files_post_first_uninstall
# check we can run it again and will work - checks both overwrite and reuse cache
result = run_via_cli(create_cmd)
coverage_env()
assert result
files_post_second_create = list(site_package.iterdir())
assert files_post_first_create == files_post_second_create
process = Popen(remove_cmd + ["pip", "wheel"])
_, __ = process.communicate()
assert not process.returncode
# pip is greedy here, removing all packages removes the site-package too
if site_package.exists():
post_run = list(site_package.iterdir())
assert not post_run, "\n".join(str(i) for i in post_run)
if sys.version_info[0:2] == (3, 4) and os.environ.get(
str("PIP_REQ_TRACKER")):
os.environ.pop(str("PIP_REQ_TRACKER"))