def activation_python(request, tmp_path_factory, special_char_name,
current_fastest):
dest = os.path.join(
ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))),
special_char_name)
cmd = [
"--without-pip", dest, "--creator", current_fastest, "-vv",
"--no-periodic-update"
]
if request.param:
cmd += ["--prompt", special_char_name]
session = cli_run(cmd)
pydoc_test = session.creator.purelib / "pydoc_test.py"
pydoc_test.write_text('"""This is pydoc_test.py"""')
yield session
if WIN_CPYTHON_2: # PY2 windows does not support unicode delete
shutil.rmtree(dest)
def get_env_var(key, as_type):
"""Get the environment variable option.
:param key: the config key requested
:param as_type: the type we would like to convert it to
:return:
"""
environ_key = ensure_str("VIRTUALENV_{}".format(key.upper()))
if os.environ.get(environ_key):
value = os.environ[environ_key]
# noinspection PyBroadException
try:
source = "env var {}".format(ensure_text(environ_key))
as_type = convert(value, as_type, source)
return as_type, source
except Exception: # note the converter already logs a warning when failures happen
pass
def __unicode__(self):
result = self.__class__.__name__
result += "("
if self.extra_search_dir:
result += "extra_search_dir={},".format(
", ".join(ensure_text(str(i)) for i in self.extra_search_dir)
)
result += "download={},".format(self.download)
for distribution in self.distributions():
if getattr(self, "no_{}".format(distribution)):
continue
result += " {}{},".format(
distribution,
"={}".format(
getattr(self, "{}_version".format(distribution), None) or "latest"
),
)
return result[:-1] + ")"
def test_app_data_pinning(tmp_path):
version = "19.1.1" if sys.version_info[0:2] == (3, 4) else "19.3.1"
result = cli_run([
ensure_text(str(tmp_path)), "--pip", version, "--activators", "",
"--seeder", "app-data"
])
code, out, err = run_cmd([
str(result.creator.script("pip")), "list",
"--disable-pip-version-check"
])
assert not code
assert not err
for line in out.splitlines():
parts = line.split()
if parts and parts[0] == "pip":
assert parts[1] == version
break
else:
assert not out
def propose_interpreters(spec, app_data):
# 1. if it's a path and exists
if spec.path is not None:
try:
os.lstat(
spec.path
) # Windows Store Python does not work with os.path.exists, but does for os.lstat
except OSError:
if spec.is_abs:
raise
else:
yield PythonInfo.from_exe(os.path.abspath(spec.path),
app_data), True
if spec.is_abs:
return
else:
# 2. otherwise try with the current
yield PythonInfo.current_system(app_data), True
# 3. otherwise fallback to platform default logic
if IS_WIN:
from .windows import propose_interpreters
for interpreter in propose_interpreters(spec, app_data):
yield interpreter, True
# finally just find on path, the path order matters (as the candidates are less easy to control by end user)
paths = get_paths()
tested_exes = set()
for pos, path in enumerate(paths):
path = ensure_text(path)
logging.debug(LazyPathDump(pos, path))
for candidate, match in possible_specs(spec):
found = check_path(candidate, path)
if found is not None:
exe = os.path.abspath(found)
if exe not in tested_exes:
tested_exes.add(exe)
interpreter = PathPythonInfo.from_exe(exe,
app_data,
raise_on_error=False)
if interpreter is not None:
yield interpreter, match
def test_create_distutils_cfg(creator, tmp_path, monkeypatch):
result = cli_run([
ensure_text(str(tmp_path / "venv")), "--activators", "", "--creator",
creator
])
app = Path(__file__).parent / "console_app"
dest = tmp_path / "console_app"
shutil.copytree(str(app), str(dest))
setup_cfg = dest / "setup.cfg"
conf = dedent("""
[install]
prefix={0}{1}prefix
install_purelib={0}{1}purelib
install_platlib={0}{1}platlib
install_headers={0}{1}headers
install_scripts={0}{1}scripts
install_data={0}{1}data
""".format(tmp_path, os.sep))
setup_cfg.write_text(setup_cfg.read_text() + conf)
monkeypatch.chdir(
dest
) # distutils will read the setup.cfg from the cwd, so change to that
install_demo_cmd = [
str(result.creator.script("pip")),
"--disable-pip-version-check",
"install",
str(dest),
"--no-use-pep517",
"-vv",
]
subprocess.check_call(install_demo_cmd)
magic = result.creator.script(
"magic") # console scripts are created in the right location
assert magic.exists()
package_folder = result.creator.purelib / "demo" # prefix is set to the virtualenv prefix for install
assert package_folder.exists(), list_files(str(tmp_path))
def test_py_pyc_missing(tmp_path, mocker, session_app_data, py, pyc):
"""Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)"""
previous = Python2.from_stdlib
def from_stdlib(mappings, name):
path, to, exists = previous(mappings, name)
if name.endswith("py"):
exists = py
elif name.endswith("pyc"):
exists = pyc
return path, to, exists
mocker.patch.object(Python2, "from_stdlib", side_effect=from_stdlib)
result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", "", "-vv"])
py_at = Python2.from_stdlib(Python2.mappings(CURRENT), "os.py")[1](result.creator, Path("os.py"))
py = pyc is False or py # if pyc is False we fallback to serve the py, which will exist (as we only mock the check)
assert py_at.exists() is py
pyc_at = Python2.from_stdlib(Python2.mappings(CURRENT), "osc.py")[1](result.creator, Path("os.pyc"))
assert pyc_at.exists() is pyc
def _install(name, wheel):
try:
logging.debug(
"install %s from wheel %s via %s",
name,
wheel,
installer_class.__name__,
)
key = Path(installer_class.__name__) / wheel.path.stem
wheel_img = self.app_data.wheel_image(
creator.interpreter.version_release_str, key)
installer = installer_class(wheel.path, creator, wheel_img)
with _CountedFileLock(
ensure_text(
str(wheel_img.parent /
"{}.lock".format(wheel_img.name)))):
if not installer.has_image():
installer.build_image()
installer.install(creator.interpreter.version_info)
except Exception: # noqa
exceptions[name] = sys.exc_info()
def test_pyc_only(tmp_path, mocker, session_app_data):
"""Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)"""
interpreter = PythonInfo.from_exe(sys.executable, session_app_data)
host_pyc, _, host_pyc_exists = Python2.from_stdlib(Python2.mappings(interpreter), "os.pyc")
if not host_pyc_exists:
pytest.skip("missing system os.pyc at {}".format(host_pyc))
previous = Python2.from_stdlib
def from_stdlib(mappings, name):
path, to, exists = previous(mappings, name)
if name.endswith(".py"):
exists = False
return path, to, exists
mocker.patch.object(Python2, "from_stdlib", side_effect=from_stdlib)
result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", ""])
assert not (result.creator.stdlib / "os.py").exists()
assert (result.creator.stdlib / "os.pyc").exists()
assert "os.pyc" in result.creator.debug["os"]
def test_venv_fails_not_inline(tmp_path, capsys, mocker):
def _session_via_cli(args, options=None):
session = session_via_cli(args, options)
assert session.creator.can_be_inline is False
return session
mocker.patch("virtualenv.run.session_via_cli", side_effect=_session_via_cli)
before = tmp_path.stat().st_mode
cfg_path = tmp_path / "pyvenv.cfg"
cfg_path.write_text(ensure_text(""))
cfg = str(cfg_path)
try:
os.chmod(cfg, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
cmd = ["-p", str(CURRENT.executable), str(tmp_path), "--without-pip", "--creator", "venv"]
with pytest.raises(SystemExit) as context:
run(cmd)
assert context.value.code != 0
finally:
os.chmod(cfg, before)
out, err = capsys.readouterr()
assert "subprocess call failed for" in out, out
assert "Error:" in err, err
def test_get_site_packages(tmp_path):
case_sensitive = fs_is_case_sensitive()
session = cli_run([ensure_text(str(tmp_path))])
env_site_packages = [
str(session.creator.purelib),
str(session.creator.platlib)
]
out = subprocess.check_output(
[
str(session.creator.exe), "-c",
r"import site; print(site.getsitepackages())"
],
universal_newlines=True,
)
site_packages = ast.literal_eval(out)
if not case_sensitive:
env_site_packages = [x.lower() for x in env_site_packages]
site_packages = [x.lower() for x in site_packages]
for env_site_package in env_site_packages:
assert env_site_package in site_packages
def assert_output(self, out, raw, tmp_path):
out = [literal_eval(i) for i in out]
assert out[0] is None # start with VIRTUAL_ENV None
prev_path = out[1]
prev_sys_path = out[2]
assert out[3] == str(self._creator.dest) # VIRTUAL_ENV now points to the virtual env folder
new_path = out[4] # PATH now starts with bin path of current
assert ([str(self._creator.bin_dir)] + prev_path) == new_path
# sys path contains the site package at its start
new_sys_path = out[5]
new_lib_paths = {ensure_text(j) if WIN_CPYTHON_2 else j for j in {str(i) for i in self._creator.libs}}
assert prev_sys_path == new_sys_path[len(new_lib_paths) :]
assert new_lib_paths == set(new_sys_path[: len(new_lib_paths)])
# manage to import from activate site package
dest = self.norm_path(self._creator.purelib / "pydoc_test.py")
found = self.norm_path(out[6].decode(sys.getfilesystemencoding()) if WIN_CPYTHON_2 else out[6])
assert found.startswith(dest)
def test_create_distutils_cfg(creator, tmp_path, monkeypatch):
cmd = [
ensure_text(str(tmp_path)),
"--activators",
"",
"--creator",
creator,
]
result = cli_run(cmd)
app = Path(__file__).parent / "console_app"
dest = tmp_path / "console_app"
shutil.copytree(str(app), str(dest))
setup_cfg = dest / "setup.cfg"
conf = dedent("""
[install]
prefix={}/a
install_scripts={}/b
""").format(tmp_path, tmp_path)
setup_cfg.write_text(setup_cfg.read_text() + conf)
monkeypatch.chdir(
dest
) # distutils will read the setup.cfg from the cwd, so change to that
install_demo_cmd = [
str(result.creator.script("pip")), "install",
str(dest), "--no-use-pep517"
]
subprocess.check_call(install_demo_cmd)
magic = result.creator.script(
"magic") # console scripts are created in the right location
assert magic.exists()
package_folder = result.creator.platlib / "demo" # prefix is set to the virtualenv prefix for install
assert package_folder.exists()
def get_env_debug_info(env_exe, debug_script, app_data):
env = os.environ.copy()
env.pop(str("PYTHONPATH"), None)
with ensure_file_on_disk(debug_script, app_data) as debug_script:
cmd = [str(env_exe), str(debug_script)]
if WIN_CPYTHON_2:
cmd = [ensure_text(i) for i in cmd]
logging.debug(str("debug via %r"), LogCmd(cmd))
code, out, err = run_cmd(cmd)
# noinspection PyBroadException
try:
if code != 0:
result = literal_eval(out)
else:
result = json.loads(out)
if err:
result["err"] = err
except Exception as exception:
return {"out": out, "err": err, "returncode": code, "exception": repr(exception)}
if "sys" in result and "path" in result["sys"]:
del result["sys"]["path"][0]
return result
def test_pth_in_site_vs_PYTHONPATH(tmp_path):
session = cli_run([ensure_text(str(tmp_path))])
site_packages = str(session.creator.purelib)
# install test.pth that sets sys.testpth='ok'
with open(os.path.join(site_packages, "test.pth"), "w") as f:
f.write('import sys; sys.testpth="ok"\n')
# verify that test.pth is activated when interpreter is run
out = subprocess.check_output(
[str(session.creator.exe), "-c", r"import sys; print(sys.testpth)"],
universal_newlines=True,
)
assert out == "ok\n"
# same with $PYTHONPATH pointing to site_packages
env = os.environ.copy()
path = [site_packages]
if "PYTHONPATH" in env:
path.append(env["PYTHONPATH"])
env["PYTHONPATH"] = os.pathsep.join(path)
out = subprocess.check_output(
[str(session.creator.exe), "-c", r"import sys; print(sys.testpth)"],
universal_newlines=True,
env=env,
)
assert out == "ok\n"
def _repr_unicode(creator, value):
py2 = creator.interpreter.version_info.major == 2
if py2: # on Python 2 we need to encode this into explicit utf-8, py3 supports unicode literals
value = ensure_text(repr(value.encode("utf-8"))[1:-1])
return value
def list_to_str(iterable):
return [ensure_text(str(i)) for i in iterable]
def test_create_no_seed(python, creator, isolated, system, coverage_env, special_name_dir, method):
dest = special_name_dir
cmd = [
"-v",
"-v",
"-p",
ensure_text(python),
ensure_text(str(dest)),
"--without-pip",
"--activators",
"",
"--creator",
creator,
"--{}".format(method),
]
if isolated == "global":
cmd.append("--system-site-packages")
result = cli_run(cmd)
coverage_env()
if IS_PYPY:
# pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits
# force a cleanup of these on system where the limit is low-ish (e.g. MacOS 256)
gc.collect()
content = list(result.creator.purelib.iterdir())
assert not content, "\n".join(ensure_text(str(i)) for i in content)
assert result.creator.env_name == ensure_text(dest.name)
debug = result.creator.debug
sys_path = cleanup_sys_path(debug["sys"]["path"])
system_sys_path = cleanup_sys_path(system["sys"]["path"])
our_paths = set(sys_path) - set(system_sys_path)
our_paths_repr = "\n".join(ensure_text(repr(i)) for i in our_paths)
# ensure we have at least one extra path added
assert len(our_paths) >= 1, our_paths_repr
# ensure all additional paths are related to the virtual environment
for path in our_paths:
msg = "\n{}\ndoes not start with {}\nhas:\n{}".format(
ensure_text(str(path)), ensure_text(str(dest)), "\n".join(ensure_text(str(p)) for p in system_sys_path),
)
assert str(path).startswith(str(dest)), msg
# ensure there's at least a site-packages folder as part of the virtual environment added
assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr
# ensure the global site package is added or not, depending on flag
last_from_system_path = next(j for j in reversed(system_sys_path) if str(j).startswith(system["sys"]["prefix"]))
if isolated == "isolated":
assert last_from_system_path not in sys_path, "last from system sys path {} is in venv sys path:\n{}".format(
ensure_text(str(last_from_system_path)), "\n".join(ensure_text(str(j)) for j in sys_path)
)
else:
common = []
for left, right in zip(reversed(system_sys_path), reversed(sys_path)):
if left == right:
common.append(left)
else:
break
def list_to_str(iterable):
return [ensure_text(str(i)) for i in iterable]
assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path)))
# test that the python executables in the bin directory are either:
# - files
# - absolute symlinks outside of the venv
# - relative symlinks inside of the venv
if sys.platform == "win32":
exes = ("python.exe",)
else:
exes = ("python", "python{}".format(*sys.version_info), "python{}.{}".format(*sys.version_info))
# pypy3<=7.3: https://bitbucket.org/pypy/pypy/pull-requests/697
if IS_PYPY and CURRENT.pypy_version_info[:3] <= [7, 3, 0] and creator == "venv":
exes = exes[:-1]
for exe in exes:
exe_path = result.creator.bin_dir / exe
assert exe_path.exists()
if not exe_path.is_symlink(): # option 1: a real file
continue # it was a file
link = os.readlink(str(exe_path))
if not os.path.isabs(link): # option 2: a relative symlink
continue
# option 3: an absolute symlink, should point outside the venv
assert not link.startswith(str(result.creator.dest))
def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest,
copies):
current = PythonInfo.current_system()
bundle_ver = BUNDLE_SUPPORT[current.version_release_str]
create_cmd = [
ensure_text(
str(tmp_path / "en v")
), # space in the name to ensure generated scripts work when path has space
"--seeder",
"app-data",
"--extra-search-dir",
ensure_text(str(BUNDLE_FOLDER)),
"--download",
"--pip",
bundle_ver["pip"].split("-")[1],
"--setuptools",
bundle_ver["setuptools"].split("-")[1],
"--clear-app-data",
"--creator",
current_fastest,
"-vv",
]
if not copies:
create_cmd.append("--symlink-app-data")
result = cli_run(create_cmd)
coverage_env()
assert result
# uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env
site_package = result.creator.purelib
pip = site_package / "pip"
setuptools = site_package / "setuptools"
files_post_first_create = list(site_package.iterdir())
assert pip in files_post_first_create
assert setuptools in files_post_first_create
for pip_exe in [
result.creator.script_dir /
"pip{}{}".format(suffix, result.creator.exe.suffix) for suffix in (
"",
"{}".format(current.version_info.major),
"{}.{}".format(current.version_info.major,
current.version_info.minor),
"-{}.{}".format(current.version_info.major,
current.version_info.minor),
)
]:
assert pip_exe.exists()
process = Popen([
ensure_text(str(pip_exe)), "--version",
"--disable-pip-version-check"
])
_, __ = process.communicate()
assert not process.returncode
remove_cmd = [
str(result.creator.script("pip")),
"--verbose",
"--disable-pip-version-check",
"uninstall",
"-y",
"setuptools",
]
process = Popen(remove_cmd)
_, __ = process.communicate()
assert not process.returncode
assert site_package.exists()
files_post_first_uninstall = list(site_package.iterdir())
assert pip in files_post_first_uninstall
assert setuptools not in files_post_first_uninstall
# check we can run it again and will work - checks both overwrite and reuse cache
result = cli_run(create_cmd)
coverage_env()
assert result
files_post_second_create = list(site_package.iterdir())
assert files_post_first_create == files_post_second_create
# Windows does not allow removing a executable while running it, so when uninstalling pip we need to do it via
# python -m pip
remove_cmd = [str(result.creator.exe), "-m", "pip"] + remove_cmd[1:]
process = Popen(remove_cmd + ["pip", "wheel"])
_, __ = process.communicate()
assert not process.returncode
# pip is greedy here, removing all packages removes the site-package too
if site_package.exists():
purelib = result.creator.purelib
patch_files = {
purelib / "{}.{}".format("_virtualenv", i)
for i in ("py", "pyc", "pth")
}
patch_files.add(purelib / "__pycache__")
post_run = set(site_package.iterdir()) - patch_files
assert not post_run, "\n".join(str(i) for i in post_run)
if sys.version_info[0:2] == (3, 4) and os.environ.get(
str("PIP_REQ_TRACKER")):
os.environ.pop(str("PIP_REQ_TRACKER"))
def __unicode__(self):
return ensure_text(self._path)
def __repr__(self):
return ensure_str("Path({})".format(ensure_text(self._path)))
session_app_data):
caplog.set_level(logging.DEBUG)
current = PythonInfo.current_system(session_app_data)
core = "somethingVeryCryptic{}".format(".".join(
str(i) for i in current.version_info[0:3]))
name = "somethingVeryCryptic"
if case == "lower":
name = name.lower()
elif case == "upper":
name = name.upper()
exe_name = "{}{}{}".format(name, current.version_info.major,
".exe" if sys.platform == "win32" else "")
target = tmp_path / current.distutils_install["scripts"]
target.mkdir(parents=True)
executable = target / exe_name
os.symlink(sys.executable, ensure_text(str(executable)))
pyvenv_cfg = Path(sys.executable).parents[1] / "pyvenv.cfg"
if pyvenv_cfg.exists():
(target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes())
new_path = os.pathsep.join(
[str(target)] + os.environ.get(str("PATH"), str("")).split(os.pathsep))
monkeypatch.setenv(str("PATH"), new_path)
interpreter = get_interpreter(core, [])
assert interpreter is not None
def test_discovery_via_path_not_found(tmp_path, monkeypatch):
monkeypatch.setenv(str("PATH"), str(tmp_path))
interpreter = get_interpreter(uuid4().hex, [])
assert interpreter is None
def __str__(self):
return "{}{} to {}".format(
"directory " if self.src.is_dir() else "",
ensure_text(str(self.src)),
ensure_text(str(self.dest)),
)
def norm(src):
return ensure_text(str(src))
def ensure_dir(path):
if not path.exists():
logging.debug("create folder %s", ensure_text(str(path)))
os.makedirs(norm(path))
def _records_text(self, files):
record_data = "\n".join("{},,".format(
os.path.relpath(ensure_text(str(rec)),
ensure_text(str(self._image_dir))))
for rec in files)
return record_data
def _args(self):
return [
("dest", ensure_text(str(self.dest))),
("clear", self.clear),
("no_vcs_ignore", self.no_vcs_ignore),
]
def write(self, content):
folder = self.file.parent
folder.mkdir(parents=True, exist_ok=True)
self.file.write_text(ensure_text(json.dumps(content, sort_keys=True, indent=2)))
logging.debug("wrote {} at %s".format(self.msg), *self.msg_args)
def _args(self):
return [
("dest", ensure_text(str(self.dest))),
("clear", self.clear),
]
def activate_call(self, script):
cmd = self.quote(ensure_text(str(self.activate_cmd)))
scr = self.quote(ensure_text(str(script)))
return "{} {}".format(cmd, scr).strip()