def test_lockfile_validation(rule_runner: RuleRunner) -> None:
"""Check that we properly load and validate lockfile metadata for both types of locks.
Note that we don't exhaustively test every source of lockfile failure nor the different options
for `--invalid-lockfile-behavior`, as those are already tested in pex_requirements_test.py.
"""
# We create a lockfile that claims it works with no requirements. It should fail when we try
# to build a PEX with a requirement.
lock_content = PythonLockfileMetadata.new(InterpreterConstraints(),
set()).add_header_to_lockfile(
b"",
regenerate_command="regen",
delimeter="#")
rule_runner.write_files({"lock.txt": lock_content.decode()})
lockfile = Lockfile(
"lock.txt",
file_path_description_of_origin="a test",
resolve_name="a",
)
with engine_error(InvalidLockfileError):
create_pex_and_get_all_data(rule_runner,
requirements=EntireLockfile(
lockfile, ("ansicolors", )))
lockfile_content = LockfileContent(
FileContent("lock.txt", lock_content),
resolve_name="a",
)
with engine_error(InvalidLockfileError):
create_pex_and_get_all_data(rule_runner,
requirements=EntireLockfile(
lockfile_content, ("ansicolors", )))
def test_pex_execution(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex],
internal_only: bool) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest((
FileContent("main.py", b'print("from main")'),
FileContent("subdir/sub.py", b'print("from sub")'),
)),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
internal_only=internal_only,
main=EntryPoint("main"),
sources=sources,
)
assert "pex" not in pex_data.files
assert "main.py" in pex_data.files
assert "subdir/sub.py" in pex_data.files
# This should run the Pex using the same interpreter used to create it. We must set the `PATH`
# so that the shebang works.
pex_exe = (f"./{pex_data.sandbox_path}" if pex_data.is_zipapp else
os.path.join(pex_data.sandbox_path, "__main__.py"))
process = Process(
argv=(pex_exe, ),
env={"PATH": os.getenv("PATH", "")},
input_digest=pex_data.pex.digest,
description="Run the pex and make sure it works",
)
result = rule_runner.request(ProcessResult, [process])
assert result.stdout == b"from main\n"
def test_venv_pex_resolve_info(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex]) -> None:
constraints = [
"requests==2.23.0",
"certifi==2020.12.5",
"chardet==3.0.4",
"idna==2.10",
"urllib3==1.25.11",
]
rule_runner.write_files({"constraints.txt": "\n".join(constraints)})
pex = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
requirements=PexRequirements(["requests==2.23.0"],
constraints_strings=constraints),
additional_pants_args=(
"--python-requirement-constraints=constraints.txt", ),
).pex
dists = rule_runner.request(PexResolveInfo, [pex])
assert dists[0] == PexDistributionInfo("certifi", Version("2020.12.5"),
None, ())
assert dists[1] == PexDistributionInfo("chardet", Version("3.0.4"), None,
())
assert dists[2] == PexDistributionInfo(
"idna", Version("2.10"),
SpecifierSet("!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"), ())
assert dists[3].project_name == "requests"
assert dists[3].version == Version("2.23.0")
# requires_dists is parsed from metadata written by the pex tool, and is always
# a set of valid pkg_resources.Requirements.
assert Requirement.parse(
'PySocks!=1.5.7,>=1.5.6; extra == "socks"') in dists[3].requires_dists
assert dists[4].project_name == "urllib3"
def test_additional_inputs(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex],
internal_only: bool) -> None:
# We use Pex's --sources-directory option to add an extra source file to the PEX.
# This verifies that the file was indeed provided as additional input to the pex call.
extra_src_dir = "extra_src"
data_file = os.path.join("data", "file")
data = "42"
additional_inputs = rule_runner.request(
Digest,
[
CreateDigest([
FileContent(path=os.path.join(extra_src_dir, data_file),
content=data.encode())
])
],
)
additional_pex_args = ("--sources-directory", extra_src_dir)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
internal_only=internal_only,
additional_inputs=additional_inputs,
additional_pex_args=additional_pex_args,
)
if pex_data.is_zipapp:
with zipfile.ZipFile(pex_data.local_path, "r") as zipfp:
with zipfp.open(data_file, "r") as datafp:
data_file_content = datafp.read()
else:
with open(pex_data.local_path / data_file, "rb") as datafp:
data_file_content = datafp.read()
assert data == data_file_content.decode()
def test_pex_environment(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex]) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest((FileContent(
path="main.py",
content=textwrap.dedent("""
from os import environ
print(f"LANG={environ.get('LANG')}")
print(f"ftp_proxy={environ.get('ftp_proxy')}")
""").encode(),
), )),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
additional_pants_args=(
"--subprocess-environment-env-vars=LANG", # Value should come from environment.
"--subprocess-environment-env-vars=ftp_proxy=dummyproxy",
),
interpreter_constraints=InterpreterConstraints(["CPython>=3.6"]),
env={"LANG": "es_PY.UTF-8"},
)
pex_process_type = PexProcess if isinstance(pex_data.pex,
Pex) else VenvPexProcess
process = rule_runner.request(
Process,
[
pex_process_type(
pex_data.pex,
description="Run the pex and check its reported environment",
),
],
)
result = rule_runner.request(ProcessResult, [process])
assert b"LANG=es_PY.UTF-8" in result.stdout
assert b"ftp_proxy=dummyproxy" in result.stdout
def test_platforms(rule_runner: RuleRunner) -> None:
# We use Python 2.7, rather than Python 3, to ensure that the specified platform is
# actually used.
platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"])
constraints = InterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"])
pex_data = create_pex_and_get_all_data(
rule_runner,
requirements=PexRequirements(["cryptography==2.9"]),
platforms=platforms,
interpreter_constraints=constraints,
internal_only=
False, # Internal only PEXes do not support (foreign) platforms.
)
assert any("cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp
for fp in pex_data.files)
assert not any("cryptography-2.9-cp27-cp27m-" in fp
for fp in pex_data.files)
assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_data.files)
# NB: Platforms override interpreter constraints.
assert pex_data.info["interpreter_constraints"] == []
def test_pex_working_directory(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex]) -> None:
named_caches_dir = rule_runner.request(GlobalOptions, []).named_caches_dir
sources = rule_runner.request(
Digest,
[
CreateDigest((FileContent(
path="main.py",
content=textwrap.dedent("""
import os
cwd = os.getcwd()
print(f"CWD: {cwd}")
for path, dirs, _ in os.walk(cwd):
for name in dirs:
print(f"DIR: {os.path.relpath(os.path.join(path, name), cwd)}")
""").encode(),
), )),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
interpreter_constraints=InterpreterConstraints(["CPython>=3.6"]),
)
pex_process_type = PexProcess if isinstance(pex_data.pex,
Pex) else VenvPexProcess
dirpath = "foo/bar/baz"
runtime_files = rule_runner.request(
Digest, [CreateDigest([Directory(path=dirpath)])])
dirpath_parts = os.path.split(dirpath)
for i in range(0, len(dirpath_parts)):
working_dir = os.path.join(*dirpath_parts[:i]) if i > 0 else None
expected_subdir = os.path.join(
*dirpath_parts[i:]) if i < len(dirpath_parts) else None
process = rule_runner.request(
Process,
[
pex_process_type(
pex_data.pex,
description="Run the pex and check its cwd",
working_directory=working_dir,
input_digest=runtime_files,
# We skip the process cache for this PEX to ensure that it re-runs.
cache_scope=ProcessCacheScope.PER_SESSION,
)
],
)
# For VenvPexes, run the PEX twice while clearing the venv dir in between. This emulates
# situations where a PEX creation hits the process cache, while venv seeding misses the PEX
# cache.
if isinstance(pex_data.pex, VenvPex):
# Request once to ensure that the directory is seeded, and then start a new session so
# that the second run happens as well.
_ = rule_runner.request(ProcessResult, [process])
rule_runner.new_session("re-run-for-venv-pex")
rule_runner.set_options(
["--backend-packages=pants.backend.python"],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
# Clear the cache.
venv_dir = os.path.join(named_caches_dir, "pex_root",
pex_data.pex.venv_rel_dir)
assert os.path.isdir(venv_dir)
safe_rmtree(venv_dir)
result = rule_runner.request(ProcessResult, [process])
output_str = result.stdout.decode()
mo = re.search(r"CWD: (.*)\n", output_str)
assert mo is not None
reported_cwd = mo.group(1)
if working_dir:
assert reported_cwd.endswith(working_dir)
if expected_subdir:
assert f"DIR: {expected_subdir}" in output_str