def test_pex_execution(rule_runner: RuleRunner) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest((
FileContent("main.py", b'print("from main")'),
FileContent("subdir/sub.py", b'print("from sub")'),
)),
],
)
pex_output = create_pex_and_get_all_data(rule_runner,
main=EntryPoint("main"),
sources=sources)
pex_files = pex_output["files"]
assert "pex" not in pex_files
assert "main.py" in pex_files
assert "subdir/sub.py" in pex_files
# This should run the Pex using the same interpreter used to create it. We must set the `PATH` so that the shebang
# works.
process = Process(
argv=("./test.pex", ),
env={"PATH": os.getenv("PATH", "")},
input_digest=pex_output["pex"].digest,
description="Run the pex and make sure it works",
)
result = rule_runner.request(ProcessResult, [process])
assert result.stdout == b"from main\n"
def test_pex_execution(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex],
internal_only: bool) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest((
FileContent("main.py", b'print("from main")'),
FileContent("subdir/sub.py", b'print("from sub")'),
)),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
internal_only=internal_only,
main=EntryPoint("main"),
sources=sources,
)
assert "pex" not in pex_data.files
assert "main.py" in pex_data.files
assert "subdir/sub.py" in pex_data.files
# This should run the Pex using the same interpreter used to create it. We must set the `PATH`
# so that the shebang works.
pex_exe = (f"./{pex_data.sandbox_path}" if pex_data.is_zipapp else
os.path.join(pex_data.sandbox_path, "__main__.py"))
process = Process(
argv=(pex_exe, ),
env={"PATH": os.getenv("PATH", "")},
input_digest=pex_data.pex.digest,
description="Run the pex and make sure it works",
)
result = rule_runner.request(ProcessResult, [process])
assert result.stdout == b"from main\n"
async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup:
parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py")
if not parser_script_content:
raise ValueError("Unable to find source to hcl2_parser.py wrapper script.")
parser_content = FileContent(
path="__pants_tf_parser.py",
content=parser_script_content,
is_executable=True,
)
parser_digest = await Get(
Digest,
CreateDigest([parser_content]),
)
parser_pex = await Get(
VenvPex,
PexRequest(
output_filename="tf_parser.pex",
internal_only=True,
requirements=hcl2_parser.pex_requirements(),
interpreter_constraints=hcl2_parser.interpreter_constraints,
main=EntryPoint(PurePath(parser_content.path).stem),
sources=parser_digest,
),
)
return ParserSetup(parser_pex)
async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:
parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)
if not parser_script_content:
raise ValueError(
"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."
)
parser_content = FileContent(
path="__pants_df_parser.py",
content=parser_script_content,
is_executable=True,
)
parser_digest = await Get(Digest, CreateDigest([parser_content]))
parser_pex = await Get(
VenvPex,
PexRequest(
output_filename="dockerfile_parser.pex",
internal_only=True,
requirements=dockerfile_parser.pex_requirements(),
interpreter_constraints=dockerfile_parser.interpreter_constraints,
main=EntryPoint(PurePath(parser_content.path).stem),
sources=parser_digest,
),
)
return ParserSetup(parser_pex)
def test_pex_binary_validation() -> None:
def create_tgt(*, script: str | None = None, entry_point: str | None = None) -> PexBinary:
return PexBinary(
{PexScriptField.alias: script, PexEntryPointField.alias: entry_point},
Address("", target_name="t"),
)
with pytest.raises(InvalidTargetException):
create_tgt(script="foo", entry_point="foo")
assert create_tgt(script="foo")[PexScriptField].value == ConsoleScript("foo")
assert create_tgt(entry_point="foo")[PexEntryPointField].value == EntryPoint("foo")
def main(self) -> MainSpecification:
is_default_console_script = self.options.is_default("console_script")
is_default_entry_point = self.options.is_default("entry_point")
if not is_default_console_script and not is_default_entry_point:
raise OptionsError(
f"Both [{self.scope}].console-script={self.options.console_script} and "
f"[{self.scope}].entry-point={self.options.entry_point} are configured but these "
f"options are mutually exclusive. Please pick one.")
if not is_default_console_script:
return ConsoleScript(cast(str, self.options.console_script))
if not is_default_entry_point:
return EntryPoint.parse(cast(str, self.options.entry_point))
return self.default_main
def test_pex_environment(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex]) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest(
(
FileContent(
path="main.py",
content=textwrap.dedent(
"""
from os import environ
print(f"LANG={environ.get('LANG')}")
print(f"ftp_proxy={environ.get('ftp_proxy')}")
"""
).encode(),
),
)
),
],
)
pex_output = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
additional_pants_args=(
"--subprocess-environment-env-vars=LANG", # Value should come from environment.
"--subprocess-environment-env-vars=ftp_proxy=dummyproxy",
),
interpreter_constraints=PexInterpreterConstraints(["CPython>=3.6"]),
env={"LANG": "es_PY.UTF-8"},
)
pex = pex_output["pex"]
pex_process_type = PexProcess if isinstance(pex, Pex) else VenvPexProcess
process = rule_runner.request(
Process,
[
pex_process_type(
pex,
description="Run the pex and check its reported environment",
),
],
)
result = rule_runner.request(ProcessResult, [process])
assert b"LANG=es_PY.UTF-8" in result.stdout
assert b"ftp_proxy=dummyproxy" in result.stdout
def main(self) -> MainSpecification:
is_default_console_script = self.options.is_default("console_script")
is_default_entry_point = self.options.is_default("entry_point")
if not is_default_console_script and not is_default_entry_point:
raise OptionsError(
softwrap(f"""
Both [{self.options_scope}].console-script={self.console_script} and
[{self.options_scope}].entry-point={self.entry_point} are configured
but these options are mutually exclusive. Please pick one.
"""))
if not is_default_console_script:
assert self.console_script is not None
return ConsoleScript(self.console_script)
if not is_default_entry_point:
assert self.entry_point is not None
return EntryPoint.parse(self.entry_point)
return self.default_main
class SetuptoolsSCM(PythonToolBase):
options_scope = "setuptools-scm"
help = (
"A tool for generating versions from VCS metadata (https://github.com/pypa/setuptools_scm)."
)
default_version = "setuptools-scm==6.4.2"
default_main = EntryPoint("setuptools_scm")
register_interpreter_constraints = True
default_interpreter_constraints = ["CPython>=3.7,<4"]
register_lockfile = True
default_lockfile_resource = ("pants.backend.python.subsystems",
"setuptools_scm.lock")
default_lockfile_path = "src/python/pants/backend/python/subsystems/setuptools_scm.lock"
default_lockfile_url = git_url(default_lockfile_path)
def test_entry_point_validation(caplog) -> None:
addr = Address("src/python/project")
with pytest.raises(InvalidFieldException):
PexEntryPointField(" ", addr)
with pytest.raises(InvalidFieldException):
PexEntryPointField("modue:func:who_knows_what_this_is", addr)
with pytest.raises(InvalidFieldException):
PexEntryPointField(":func", addr)
ep = "custom.entry_point:"
with caplog.at_level(logging.WARNING):
assert EntryPoint("custom.entry_point") == PexEntryPointField(ep, addr).value
assert len(caplog.record_tuples) == 1
_, levelno, message = caplog.record_tuples[0]
assert logging.WARNING == levelno
assert ep in message
assert str(addr) in message
def test_resolve_pex_binary_entry_point() -> None:
rule_runner = RuleRunner(
rules=[
resolve_pex_entry_point,
QueryRule(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest]),
]
)
def assert_resolved(
*, entry_point: str | None, expected: EntryPoint | None, is_file: bool
) -> None:
addr = Address("src/python/project")
rule_runner.write_files(
{
"src/python/project/app.py": "",
"src/python/project/f2.py": "",
}
)
ep_field = PexEntryPointField(entry_point, addr)
result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)])
assert result.val == expected
assert result.file_name_used == is_file
# Full module provided.
assert_resolved(
entry_point="custom.entry_point", expected=EntryPoint("custom.entry_point"), is_file=False
)
assert_resolved(
entry_point="custom.entry_point:func",
expected=EntryPoint.parse("custom.entry_point:func"),
is_file=False,
)
# File names are expanded into the full module path.
assert_resolved(entry_point="app.py", expected=EntryPoint(module="project.app"), is_file=True)
assert_resolved(
entry_point="app.py:func",
expected=EntryPoint(module="project.app", function="func"),
is_file=True,
)
with pytest.raises(ExecutionError):
assert_resolved(
entry_point="doesnt_exist.py", expected=EntryPoint("doesnt matter"), is_file=True
)
# Resolving >1 file is an error.
with pytest.raises(ExecutionError):
assert_resolved(entry_point="*.py", expected=EntryPoint("doesnt matter"), is_file=True)
def test_resolve_pex_binary_entry_point() -> None:
rule_runner = RuleRunner(rules=[
resolve_pex_entry_point,
QueryRule(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest]),
])
def assert_resolved(*, entry_point: Optional[str],
expected: Optional[EntryPoint]) -> None:
addr = Address("src/python/project")
rule_runner.create_file("src/python/project/app.py")
rule_runner.create_file("src/python/project/f2.py")
ep_field = PexEntryPointField(entry_point, address=addr)
result = rule_runner.request(ResolvedPexEntryPoint,
[ResolvePexEntryPointRequest(ep_field)])
assert result.val == expected
# Full module provided.
assert_resolved(entry_point="custom.entry_point",
expected=EntryPoint("custom.entry_point"))
assert_resolved(entry_point="custom.entry_point:func",
expected=EntryPoint.parse("custom.entry_point:func"))
# File names are expanded into the full module path.
assert_resolved(entry_point="app.py",
expected=EntryPoint(module="project.app"))
assert_resolved(entry_point="app.py:func",
expected=EntryPoint(module="project.app", function="func"))
# We special case the strings `<none>` and `<None>`.
assert_resolved(entry_point="<none>", expected=None)
assert_resolved(entry_point="<None>", expected=None)
with pytest.raises(ExecutionError):
assert_resolved(entry_point="doesnt_exist.py",
expected=EntryPoint("doesnt matter"))
# Resolving >1 file is an error.
with pytest.raises(ExecutionError):
assert_resolved(entry_point="*.py",
expected=EntryPoint("doesnt matter"))
async def generate_lockfile(
req: PythonLockfileRequest,
poetry_subsystem: PoetrySubsystem,
generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
) -> PythonLockfile:
pyproject_toml = create_pyproject_toml(req.requirements, req.interpreter_constraints).encode()
pyproject_toml_digest, launcher_digest = await MultiGet(
Get(Digest, CreateDigest([FileContent("pyproject.toml", pyproject_toml)])),
Get(Digest, CreateDigest([POETRY_LAUNCHER])),
)
poetry_pex = await Get(
VenvPex,
PexRequest(
output_filename="poetry.pex",
internal_only=True,
requirements=poetry_subsystem.pex_requirements(),
interpreter_constraints=poetry_subsystem.interpreter_constraints,
main=EntryPoint(PurePath(POETRY_LAUNCHER.path).stem),
sources=launcher_digest,
),
)
# WONTFIX(#12314): Wire up Poetry to named_caches.
# WONTFIX(#12314): Wire up all the pip options like indexes.
poetry_lock_result = await Get(
ProcessResult,
VenvPexProcess(
poetry_pex,
argv=("lock",),
input_digest=pyproject_toml_digest,
output_files=("poetry.lock", "pyproject.toml"),
description=req._description or f"Generate lockfile for {req.resolve_name}",
# Instead of caching lockfile generation with LMDB, we instead use the invalidation
# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
# necessary so that our invalidation is resilient to deleting LMDB or running on a
# new machine.
#
# We disable caching with LMDB so that when you generate a lockfile, you always get
# the most up-to-date snapshot of the world. This is generally desirable and also
# necessary to avoid an awkward edge case where different developers generate different
# lockfiles even when generating at the same time. See
# https://github.com/pantsbuild/pants/issues/12591.
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
poetry_export_result = await Get(
ProcessResult,
VenvPexProcess(
poetry_pex,
argv=("export", "-o", req.lockfile_dest),
input_digest=poetry_lock_result.output_digest,
output_files=(req.lockfile_dest,),
description=(
f"Exporting Poetry lockfile to requirements.txt format for {req.resolve_name}"
),
level=LogLevel.DEBUG,
),
)
initial_lockfile_digest_contents = await Get(
DigestContents, Digest, poetry_export_result.output_digest
)
# TODO(#12314) Improve error message on `Requirement.parse`
metadata = LockfileMetadata.new(
req.interpreter_constraints,
{PipRequirement.parse(i) for i in req.requirements},
)
lockfile_with_header = metadata.add_header_to_lockfile(
initial_lockfile_digest_contents[0].content,
regenerate_command=(
generate_lockfiles_subsystem.custom_command
or req._regenerate_command
or f"./pants generate-lockfiles --resolve={req.resolve_name}"
),
)
final_lockfile_digest = await Get(
Digest, CreateDigest([FileContent(req.lockfile_dest, lockfile_with_header)])
)
return PythonLockfile(final_lockfile_digest, req.resolve_name, req.lockfile_dest)
async def resolve_python_distribution_entry_points(
request: ResolvePythonDistributionEntryPointsRequest,
) -> ResolvedPythonDistributionEntryPoints:
if request.entry_points_field:
if request.entry_points_field.value is None:
return ResolvedPythonDistributionEntryPoints()
address = request.entry_points_field.address
all_entry_points = cast(_EntryPointsDictType, request.entry_points_field.value)
elif request.provides_field:
address = request.provides_field.address
provides_field_value = cast(
_EntryPointsDictType, request.provides_field.value.kwargs.get("entry_points") or {}
)
if provides_field_value:
all_entry_points = provides_field_value
else:
return ResolvedPythonDistributionEntryPoints()
else:
return ResolvedPythonDistributionEntryPoints()
classified_entry_points = list(_classify_entry_points(all_entry_points))
# Pick out all target addresses up front, so we can use MultiGet later.
#
# This calls for a bit of trickery however (using the "y_by_x" mapping dicts), so we keep track
# of which address belongs to which entry point. I.e. the `address_by_ref` and
# `binary_entry_point_by_address` variables.
target_refs = [
entry_point_str for is_target, _, _, entry_point_str in classified_entry_points if is_target
]
# Intermediate step, as Get(Targets) returns a deduplicated set.. which breaks in case of
# multiple input refs that maps to the same target.
target_addresses = await Get(
Addresses,
UnparsedAddressInputs(
target_refs,
owning_address=address,
description_of_origin="TODO(#14468)",
),
)
address_by_ref = dict(zip(target_refs, target_addresses))
targets = await Get(Targets, Addresses, target_addresses)
# Check that we only have targets with a pex entry_point field.
for target in targets:
if not target.has_field(PexEntryPointField):
raise InvalidEntryPoint(
softwrap(
f"""
All target addresses in the entry_points field must be for pex_binary targets,
but the target {address} includes the value {target.address}, which has the
target type {target.alias}.
Alternatively, you can use a module like "project.app:main".
See {doc_url('python-distributions')}.
"""
)
)
binary_entry_points = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(target[PexEntryPointField]),
)
for target in targets
)
binary_entry_point_by_address = {
target.address: entry_point for target, entry_point in zip(targets, binary_entry_points)
}
entry_points: DefaultDict[str, Dict[str, PythonDistributionEntryPoint]] = defaultdict(dict)
# Parse refs/replace with resolved pex entry point, and validate console entry points have function.
for is_target, category, name, ref in classified_entry_points:
owner: Optional[Address] = None
if is_target:
owner = address_by_ref[ref]
entry_point = binary_entry_point_by_address[owner].val
if entry_point is None:
logger.warning(
softwrap(
f"""
The entry point {name} in {category} references a pex_binary target {ref}
which does not set `entry_point`. Skipping.
"""
)
)
continue
else:
entry_point = EntryPoint.parse(ref, f"{name} for {address} {category}")
if category in ["console_scripts", "gui_scripts"] and not entry_point.function:
url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point"
raise InvalidEntryPoint(
dedent(
f"""\
Every entry point in `{category}` for {address} must end in the format `:my_func`,
but {name} set it to {entry_point.spec!r}. For example, set
`entry_points={{"{category}": {{"{name}": "{entry_point.module}:main}} }}`.
See {url}.
"""
)
)
entry_points[category][name] = PythonDistributionEntryPoint(entry_point, owner)
return ResolvedPythonDistributionEntryPoints(
FrozenDict(
{category: FrozenDict(entry_points) for category, entry_points in entry_points.items()}
)
)
def test_pex_working_directory(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex]) -> None:
named_caches_dir = rule_runner.request(GlobalOptions, []).named_caches_dir
sources = rule_runner.request(
Digest,
[
CreateDigest((FileContent(
path="main.py",
content=textwrap.dedent("""
import os
cwd = os.getcwd()
print(f"CWD: {cwd}")
for path, dirs, _ in os.walk(cwd):
for name in dirs:
print(f"DIR: {os.path.relpath(os.path.join(path, name), cwd)}")
""").encode(),
), )),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
interpreter_constraints=InterpreterConstraints(["CPython>=3.6"]),
)
pex_process_type = PexProcess if isinstance(pex_data.pex,
Pex) else VenvPexProcess
dirpath = "foo/bar/baz"
runtime_files = rule_runner.request(
Digest, [CreateDigest([Directory(path=dirpath)])])
dirpath_parts = os.path.split(dirpath)
for i in range(0, len(dirpath_parts)):
working_dir = os.path.join(*dirpath_parts[:i]) if i > 0 else None
expected_subdir = os.path.join(
*dirpath_parts[i:]) if i < len(dirpath_parts) else None
process = rule_runner.request(
Process,
[
pex_process_type(
pex_data.pex,
description="Run the pex and check its cwd",
working_directory=working_dir,
input_digest=runtime_files,
# We skip the process cache for this PEX to ensure that it re-runs.
cache_scope=ProcessCacheScope.PER_SESSION,
)
],
)
# For VenvPexes, run the PEX twice while clearing the venv dir in between. This emulates
# situations where a PEX creation hits the process cache, while venv seeding misses the PEX
# cache.
if isinstance(pex_data.pex, VenvPex):
# Request once to ensure that the directory is seeded, and then start a new session so
# that the second run happens as well.
_ = rule_runner.request(ProcessResult, [process])
rule_runner.new_session("re-run-for-venv-pex")
rule_runner.set_options(
["--backend-packages=pants.backend.python"],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
# Clear the cache.
venv_dir = os.path.join(named_caches_dir, "pex_root",
pex_data.pex.venv_rel_dir)
assert os.path.isdir(venv_dir)
safe_rmtree(venv_dir)
result = rule_runner.request(ProcessResult, [process])
output_str = result.stdout.decode()
mo = re.search(r"CWD: (.*)\n", output_str)
assert mo is not None
reported_cwd = mo.group(1)
if working_dir:
assert reported_cwd.endswith(working_dir)
if expected_subdir:
assert f"DIR: {expected_subdir}" in output_str
async def generate_lockfile(
req: GeneratePythonLockfile,
poetry_subsystem: PoetrySubsystem,
generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
python_repos: PythonRepos,
python_setup: PythonSetup,
) -> GenerateLockfileResult:
if req.use_pex:
header_delimiter = "//"
result = await Get(
ProcessResult,
PexCliProcess(
subcommand=("lock", "create"),
extra_args=(
"--output=lock.json",
"--no-emit-warnings",
# See https://github.com/pantsbuild/pants/issues/12458. For now, we always
# generate universal locks because they have the best compatibility. We may
# want to let users change this, as `style=strict` is safer.
"--style=universal",
"--resolver-version",
"pip-2020-resolver",
# This makes diffs more readable when lockfiles change.
"--indent=2",
*python_repos.pex_args,
*python_setup.manylinux_pex_args,
*req.interpreter_constraints.generate_pex_arg_list(),
*req.requirements,
),
output_files=("lock.json", ),
description=f"Generate lockfile for {req.resolve_name}",
# Instead of caching lockfile generation with LMDB, we instead use the invalidation
# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
# necessary so that our invalidation is resilient to deleting LMDB or running on a
# new machine.
#
# We disable caching with LMDB so that when you generate a lockfile, you always get
# the most up-to-date snapshot of the world. This is generally desirable and also
# necessary to avoid an awkward edge case where different developers generate
# different lockfiles even when generating at the same time. See
# https://github.com/pantsbuild/pants/issues/12591.
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
else:
header_delimiter = "#"
await Get(MaybeWarnPythonRepos, MaybeWarnPythonReposRequest())
_pyproject_toml = create_pyproject_toml(
req.requirements, req.interpreter_constraints).encode()
_pyproject_toml_digest, _launcher_digest = await MultiGet(
Get(Digest,
CreateDigest([FileContent("pyproject.toml",
_pyproject_toml)])),
Get(Digest, CreateDigest([POETRY_LAUNCHER])),
)
_poetry_pex = await Get(
VenvPex,
PexRequest,
poetry_subsystem.to_pex_request(main=EntryPoint(
PurePath(POETRY_LAUNCHER.path).stem),
sources=_launcher_digest),
)
# WONTFIX(#12314): Wire up Poetry to named_caches.
# WONTFIX(#12314): Wire up all the pip options like indexes.
_lock_result = await Get(
ProcessResult,
VenvPexProcess(
_poetry_pex,
argv=("lock", ),
input_digest=_pyproject_toml_digest,
output_files=("poetry.lock", "pyproject.toml"),
description=f"Generate lockfile for {req.resolve_name}",
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
result = await Get(
ProcessResult,
VenvPexProcess(
_poetry_pex,
argv=("export", "-o", req.lockfile_dest),
input_digest=_lock_result.output_digest,
output_files=(req.lockfile_dest, ),
description=
(f"Exporting Poetry lockfile to requirements.txt format for {req.resolve_name}"
),
level=LogLevel.DEBUG,
),
)
initial_lockfile_digest_contents = await Get(DigestContents, Digest,
result.output_digest)
# TODO(#12314) Improve error message on `Requirement.parse`
metadata = PythonLockfileMetadata.new(
req.interpreter_constraints,
{PipRequirement.parse(i)
for i in req.requirements},
)
lockfile_with_header = metadata.add_header_to_lockfile(
initial_lockfile_digest_contents[0].content,
regenerate_command=(
generate_lockfiles_subsystem.custom_command or
f"{bin_name()} generate-lockfiles --resolve={req.resolve_name}"),
delimeter=header_delimiter,
)
final_lockfile_digest = await Get(
Digest,
CreateDigest([FileContent(req.lockfile_dest, lockfile_with_header)]))
return GenerateLockfileResult(final_lockfile_digest, req.resolve_name,
req.lockfile_dest)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])
)
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup
)
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True),
)
pytest_pex_request = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
prepared_sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)
)
# Create any assets that the test depends on through the `runtime_package_dependencies` field.
assets: Tuple[BuiltPackage, ...] = ()
unparsed_runtime_packages = (
request.field_set.runtime_package_dependencies.to_unparsed_address_inputs()
)
if unparsed_runtime_packages.values:
runtime_package_targets = await Get(
Targets, UnparsedAddressInputs, unparsed_runtime_packages
)
field_sets_per_target = await Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets),
)
assets = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in field_sets_per_target.field_sets
)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_request = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources])
)
pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet(
pytest_pex_request,
requirements_pex_request,
prepared_sources_request,
field_set_source_files_request,
)
pytest_runner_pex = await Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
# TODO(John Sirois): Switch to ConsoleScript once Pex supports discovering console
# scripts via the PEX_PATH: https://github.com/pantsbuild/pex/issues/1257
main=EntryPoint("pytest"),
internal_only=True,
pex_path=[pytest_pex, requirements_pex],
),
)
input_digest = await Get(
Digest,
MergeDigests(
(
coverage_config.digest,
prepared_sources.source_files.snapshot.digest,
*(binary.digest for binary in assets),
)
),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend(
(f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")
)
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
output_files.append(".coverage")
cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (".",)
coverage_args = [
"--cov-report=", # Turn off output.
*itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths),
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
}
extra_env.update(test_extra_env.env)
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_files=output_files,
timeout_seconds=request.field_set.timeout.calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def collect_fixture_configs(
_request: CollectFixtureConfigsRequest,
pytest: PyTest,
python_setup: PythonSetup,
test_extra_env: TestExtraEnv,
targets: Targets,
) -> CollectedJVMLockfileFixtureConfigs:
addresses = [tgt.address for tgt in targets]
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(addresses))
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet(
Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
),
Get(Pex, RequirementsPexRequest(addresses)),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets,
include_files=True,
include_resources=True),
),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(targets),
),
)
script_content = FileContent(path="collect-fixtures.py",
content=COLLECTION_SCRIPT.encode(),
is_executable=True)
script_digest = await Get(Digest, CreateDigest([script_content]))
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=EntryPoint(PurePath(script_content.path).stem),
sources=script_digest,
internal_only=True,
pex_path=[
pytest_pex,
requirements_pex,
],
),
)
config_file_dirs = list(
group_by_dir(prepared_sources.source_files.files).keys())
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(config_file_dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
pytest_config_digest = config_files.snapshot.digest
input_digest = await Get(
Digest,
MergeDigests((
prepared_sources.source_files.snapshot.digest,
pytest_config_digest,
)),
)
extra_env = {
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
}
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=[
name for name in root_sources.source_files.files
if name.endswith(".py")
],
extra_env=extra_env,
input_digest=input_digest,
output_files=("tests.json", ),
description="Collect test lockfile requirements from all tests.",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
result = await Get(ProcessResult, Process, process)
digest_contents = await Get(DigestContents, Digest, result.output_digest)
assert len(digest_contents) == 1
assert digest_contents[0].path == "tests.json"
raw_config_data = json.loads(digest_contents[0].content)
configs = []
for item in raw_config_data:
config = JVMLockfileFixtureConfig(
definition=JVMLockfileFixtureDefinition.from_kwargs(
item["kwargs"]),
test_file_path=item["test_file_path"],
)
configs.append(config)
return CollectedJVMLockfileFixtureConfigs(configs)
def test_entry_point(rule_runner: RuleRunner) -> None:
entry_point = "pydoc"
pex_info = create_pex_and_get_pex_info(rule_runner,
main=EntryPoint(entry_point))
assert pex_info["entry_point"] == entry_point
class Pylint(PythonToolBase):
options_scope = "pylint"
help = "The Pylint linter for Python code (https://www.pylint.org/)."
default_version = "pylint>=2.4.4,<2.5"
# TODO(John Sirois): Switch to ConsoleScript once Pex supports discovering console
# scripts via the PEX_PATH: https://github.com/pantsbuild/pex/issues/1257
default_main = EntryPoint("pylint")
@classmethod
def register_options(cls, register):
super().register_options(register)
register(
"--skip",
type=bool,
default=False,
help=
f"Don't use Pylint when running `{register.bootstrap.pants_bin_name} lint`",
)
register(
"--args",
type=list,
member_type=shell_str,
help=
("Arguments to pass directly to Pylint, e.g. "
f'`--{cls.options_scope}-args="--ignore=foo.py,bar.py --disable=C0330,W0311"`'
),
)
register(
"--config",
type=file_option,
default=None,
advanced=True,
help="Path to `pylintrc` or alternative Pylint config file",
)
register(
"--source-plugins",
type=list,
member_type=target_option,
advanced=True,
help=
("An optional list of `python_library` target addresses to load first-party "
"plugins.\n\nYou must set the plugin's parent directory as a source root. For "
"example, if your plugin is at `build-support/pylint/custom_plugin.py`, add "
"'build-support/pylint' to `[source].root_patterns` in `pants.toml`. This is "
"necessary for Pants to know how to tell Pylint to discover your plugin. See "
f"{docs_url('source-roots')}\n\nYou must also set `load-plugins=$module_name` in "
"your Pylint config file, and set the `[pylint].config` option in `pants.toml`."
"\n\nWhile your plugin's code can depend on other first-party code and third-party "
"requirements, all first-party dependencies of the plugin must live in the same "
"directory or a subdirectory.\n\nTo instead load third-party plugins, set the "
"option `[pylint].extra_requirements` and set the `load-plugins` option in your "
"Pylint config."),
)
@property
def skip(self) -> bool:
return cast(bool, self.options.skip)
@property
def args(self) -> List[str]:
return cast(List[str], self.options.args)
@property
def config(self) -> str | None:
return cast("str | None", self.options.config)
@property
def source_plugins(self) -> UnparsedAddressInputs:
return UnparsedAddressInputs(self.options.source_plugins,
owning_address=None)