async def prepare_python_sources(
request: PythonSourceFilesRequest, union_membership: UnionMembership
) -> PythonSourceFiles:
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in request.targets),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
)
missing_init_files = await Get(
AncestorFiles, AncestorFilesRequest("__init__.py", sources.snapshot),
)
init_injected = await Get(
Snapshot, MergeDigests((sources.snapshot.digest, missing_init_files.snapshot.digest)),
)
source_root_objs = await MultiGet(
Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(tgt))
for tgt in request.targets
if (
tgt.has_field(PythonSources)
or tgt.has_field(ResourcesSources)
or tgt.get(Sources).can_generate(PythonSources, union_membership)
or tgt.get(Sources).can_generate(ResourcesSources, union_membership)
)
)
source_root_paths = {source_root_obj.path for source_root_obj in source_root_objs}
return PythonSourceFiles(
SourceFiles(init_injected, sources.unrooted_files), tuple(sorted(source_root_paths))
)
async def create_python_binary(
field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults
) -> CreatedBinary:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
output_filename = f"{field_set.address.target_name}.pex"
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(python_binary_defaults),
)
),
)
pex = two_step_pex.pex
return CreatedBinary(digest=pex.digest, binary_name=pex.name)
def get_stripped_files_for_snapshot(
paths: List[str],
*,
args: Optional[List[str]] = None,
) -> List[str]:
input_snapshot = self.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return self.get_stripped_files(request, args=args)
def test_strip_snapshot(rule_runner: RuleRunner) -> None:
def get_stripped_files_for_snapshot(
paths: list[str],
*,
source_root_patterns: Sequence[str] = ("src/python", "src/java",
"tests/python"),
) -> list[str]:
input_snapshot = rule_runner.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return get_stripped_files(rule_runner,
request,
source_root_patterns=source_root_patterns)
# Normal source roots
assert get_stripped_files_for_snapshot(["src/python/project/example.py"
]) == ["project/example.py"]
assert (get_stripped_files_for_snapshot(
["src/python/project/example.py"], ) == ["project/example.py"])
assert get_stripped_files_for_snapshot(
["src/java/com/project/example.java"]) == ["com/project/example.java"]
assert get_stripped_files_for_snapshot([
"tests/python/project_test/example.py"
]) == ["project_test/example.py"]
# Unrecognized source root
unrecognized_source_root = "no-source-root/example.txt"
with pytest.raises(ExecutionError) as exc:
get_stripped_files_for_snapshot([unrecognized_source_root])
assert f"NoSourceRootError: No source root found for `{unrecognized_source_root}`." in str(
exc.value)
# Support for multiple source roots
file_names = [
"src/python/project/example.py", "src/java/com/project/example.java"
]
assert get_stripped_files_for_snapshot(file_names) == [
"com/project/example.java",
"project/example.py",
]
# Test a source root at the repo root. We have performance optimizations for this case
# because there is nothing to strip.
assert get_stripped_files_for_snapshot(["project/f1.py", "project/f2.py"],
source_root_patterns=["/"]) == [
"project/f1.py", "project/f2.py"
]
assert get_stripped_files_for_snapshot(["dir1/f.py", "dir2/f.py"],
source_root_patterns=["/"]) == [
"dir1/f.py", "dir2/f.py"
]
# Gracefully handle an empty snapshot
assert get_stripped_files(rule_runner, SourceFiles(EMPTY_SNAPSHOT,
())) == []
def test_strip_snapshot(self) -> None:
def get_stripped_files_for_snapshot(
paths: List[str],
*,
args: Optional[List[str]] = None,
) -> List[str]:
input_snapshot = self.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return self.get_stripped_files(request, args=args)
# Normal source roots
assert get_stripped_files_for_snapshot(
["src/python/project/example.py"]) == ["project/example.py"]
assert get_stripped_files_for_snapshot(
["src/python/project/example.py"], ) == ["project/example.py"]
assert get_stripped_files_for_snapshot([
"src/java/com/project/example.java"
]) == ["com/project/example.java"]
assert get_stripped_files_for_snapshot([
"tests/python/project_test/example.py"
]) == ["project_test/example.py"]
# Unrecognized source root
unrecognized_source_root = "no-source-root/example.txt"
with pytest.raises(ExecutionError) as exc:
get_stripped_files_for_snapshot([unrecognized_source_root])
assert f"NoSourceRootError: No source root found for `{unrecognized_source_root}`." in str(
exc.value)
# Support for multiple source roots
file_names = [
"src/python/project/example.py",
"src/java/com/project/example.java"
]
assert get_stripped_files_for_snapshot(file_names) == [
"com/project/example.java",
"project/example.py",
]
# Test a source root at the repo root. We have performance optimizations for this case
# because there is nothing to strip.
source_root_config = [f"--source-root-patterns={json.dumps(['/'])}"]
assert get_stripped_files_for_snapshot(
["project/f1.py", "project/f2.py"],
args=source_root_config,
) == ["project/f1.py", "project/f2.py"]
assert get_stripped_files_for_snapshot(
["dir1/f.py", "dir2/f.py"],
args=source_root_config,
) == ["dir1/f.py", "dir2/f.py"]
# Gracefully handle an empty snapshot
assert self.get_stripped_files(SourceFiles(EMPTY_SNAPSHOT, ())) == []
def test_build_local_dists(rule_runner: RuleRunner) -> None:
foo = PurePath("foo")
rule_runner.write_files({
foo / "BUILD":
dedent("""
python_sources()
python_distribution(
name = "dist",
dependencies = [":foo"],
provides = python_artifact(name="foo", version="9.8.7"),
sdist = False,
generate_setup = False,
)
"""),
foo / "bar.py":
"BAR = 42",
foo / "setup.py":
dedent("""
from setuptools import setup
setup(name="foo", version="9.8.7", packages=["foo"], package_dir={"foo": "."},)
"""),
})
rule_runner.set_options([], env_inherit={"PATH"})
sources_digest = rule_runner.request(
Digest,
[
CreateDigest([
FileContent("srcroot/foo/bar.py", b""),
FileContent("srcroot/foo/qux.py", b"")
])
],
)
sources_snapshot = rule_runner.request(Snapshot, [sources_digest])
sources = PythonSourceFiles(SourceFiles(sources_snapshot, tuple()),
("srcroot", ))
request = LocalDistsPexRequest([Address("foo", target_name="dist")],
internal_only=True,
sources=sources)
result = rule_runner.request(LocalDistsPex, [request])
assert result.pex is not None
contents = rule_runner.request(DigestContents, [result.pex.digest])
whl_content = None
for content in contents:
if content.path == "local_dists.pex/.deps/foo-9.8.7-py3-none-any.whl":
whl_content = content
assert whl_content
with io.BytesIO(whl_content.content) as fp:
with zipfile.ZipFile(fp, "r") as whl:
assert "foo/bar.py" in whl.namelist()
# Check that srcroot/foo/bar.py was subtracted out, because the dist provides foo/bar.py.
assert result.remaining_sources.source_files.files == (
"srcroot/foo/qux.py", )
def get_stripped_files_for_snapshot(
paths: list[str],
*,
source_root_patterns: Sequence[str] = ("src/python", "src/java",
"tests/python"),
) -> list[str]:
input_snapshot = rule_runner.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return get_stripped_files(rule_runner,
request,
source_root_patterns=source_root_patterns)
def __init__(
self,
addresses: Iterable[Address],
*,
interpreter_constraints:
InterpreterConstraints = InterpreterConstraints(),
sources: PythonSourceFiles = PythonSourceFiles(
SourceFiles(EMPTY_SNAPSHOT, tuple()), tuple()),
) -> None:
self.addresses = Addresses(addresses)
self.interpreter_constraints = interpreter_constraints
self.sources = sources
async def build_local_dists(
request: LocalDistsPexRequest,
) -> LocalDistsPex:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses))
applicable_targets = [
tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt)
]
local_dists_wheels = await MultiGet(
Get(LocalDistWheels, PythonDistributionFieldSet, PythonDistributionFieldSet.create(target))
for target in applicable_targets
)
# The primary use-case of the "local dists" feature is to support consuming native extensions
# as wheels without having to publish them first.
# It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
# reason about possible sys.path collisions between the in-repo sources and whatever the
# sdist will place on the sys.path when it's installed.
# So for now we simply ignore sdists, with a warning if necessary.
provided_files: set[str] = set()
wheels: list[str] = []
wheels_digests = []
for local_dist_wheels in local_dists_wheels:
wheels.extend(local_dist_wheels.wheel_paths)
wheels_digests.append(local_dist_wheels.wheels_digest)
provided_files.update(local_dist_wheels.provided_files)
wheels_digest = await Get(Digest, MergeDigests(wheels_digests))
dists_pex = await Get(
Pex,
PexRequest(
output_filename="local_dists.pex",
requirements=PexRequirements(wheels),
interpreter_constraints=request.interpreter_constraints,
additional_inputs=wheels_digest,
internal_only=request.internal_only,
additional_args=["--intransitive"],
),
)
if not wheels:
# The source calculations below are not (always) cheap, so we skip them if no wheels were
# produced. See https://github.com/pantsbuild/pants/issues/14561 for one possible approach
# to sharing the cost of these calculations.
return LocalDistsPex(dists_pex, request.sources)
# We check source roots in reverse lexicographic order,
# so we'll find the innermost root that matches.
source_roots = sorted(request.sources.source_roots, reverse=True)
remaining_sources = set(request.sources.source_files.files)
unrooted_files_set = set(request.sources.source_files.unrooted_files)
for source in request.sources.source_files.files:
if source not in unrooted_files_set:
for source_root in source_roots:
source_relpath = fast_relpath_optional(source, source_root)
if source_relpath is not None and source_relpath in provided_files:
remaining_sources.remove(source)
remaining_sources_snapshot = await Get(
Snapshot,
DigestSubset(
request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources))
),
)
subtracted_sources = PythonSourceFiles(
SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files),
request.sources.source_roots,
)
return LocalDistsPex(dists_pex, subtracted_sources)
async def build_local_dists(request: LocalDistsPexRequest, ) -> LocalDistsPex:
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(request.addresses))
applicable_targets = [
tgt for tgt in transitive_targets.closure
if PythonDistributionFieldSet.is_applicable(tgt)
]
python_dist_field_sets = [
PythonDistributionFieldSet.create(target)
for target in applicable_targets
]
dists = await MultiGet([
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in python_dist_field_sets
])
# The primary use-case of the "local dists" feature is to support consuming native extensions
# as wheels without having to publish them first.
# It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
# reason about possible sys.path collisions between the in-repo sources and whatever the
# sdist will place on the sys.path when it's installed.
# So for now we simply ignore sdists, with a warning if necessary.
provided_files = set()
wheels = []
all_contents = await MultiGet(
Get(DigestContents, Digest, dist.digest) for dist in dists)
for dist, contents, tgt in zip(dists, all_contents, applicable_targets):
artifacts = set((a.relpath or "") for a in dist.artifacts)
# A given local dist might build a wheel and an sdist (and maybe other artifacts -
# we don't know what setup command was run...)
# As long as there is a wheel, we can ignore the other artifacts.
wheel = next((art for art in artifacts if art.endswith(".whl")), None)
if wheel:
wheel_content = next(content for content in contents
if content.path == wheel)
wheels.append(wheel)
buf = BytesIO()
buf.write(wheel_content.content)
buf.seek(0)
with zipfile.ZipFile(buf) as zf:
provided_files.update(zf.namelist())
else:
logger.warning(
f"Encountered a dependency on the {tgt.alias} target at {tgt.address.spec}, but "
"this target does not produce a Python wheel artifact. Therefore this target's "
"code will be used directly from sources, without a distribution being built, "
"and therefore any native extensions in it will not be built.\n\n"
f"See {doc_url('python-distributions')} for details on how to set up a {tgt.alias} "
"target to produce a wheel.")
dists_digest = await Get(Digest,
MergeDigests([dist.digest for dist in dists]))
wheels_digest = await Get(
Digest, DigestSubset(dists_digest, PathGlobs(["**/*.whl"])))
dists_pex = await Get(
Pex,
PexRequest(
output_filename="local_dists.pex",
requirements=PexRequirements(wheels),
interpreter_constraints=request.interpreter_constraints,
additional_inputs=wheels_digest,
internal_only=True,
),
)
# We check source roots in reverse lexicographic order,
# so we'll find the innermost root that matches.
source_roots = list(reversed(sorted(request.sources.source_roots)))
remaining_sources = set(request.sources.source_files.files)
unrooted_files_set = set(request.sources.source_files.unrooted_files)
for source in request.sources.source_files.files:
if source not in unrooted_files_set:
for source_root in source_roots:
if (source.startswith(source_root) and os.path.relpath(
source, source_root) in provided_files):
remaining_sources.remove(source)
remaining_sources_snapshot = await Get(
Snapshot,
DigestSubset(request.sources.source_files.snapshot.digest,
PathGlobs(sorted(remaining_sources))),
)
subtracted_sources = PythonSourceFiles(
SourceFiles(remaining_sources_snapshot,
request.sources.source_files.unrooted_files),
request.sources.source_roots,
)
return LocalDistsPex(dists_pex, subtracted_sources)
async def prepare_python_sources(
request: PythonSourceFilesRequest,
union_membership: UnionMembership) -> PythonSourceFiles:
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(SourcesField) for tgt in request.targets),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
)
missing_init_files = await Get(
AncestorFiles,
AncestorFilesRequest(input_files=sources.snapshot.files,
requested=("__init__.py", "__init__.pyi")),
)
init_injected = await Get(
Snapshot,
MergeDigests(
(sources.snapshot.digest, missing_init_files.snapshot.digest)))
# Codegen is able to generate code in any arbitrary location, unlike sources normally being
# rooted under the target definition. To determine source roots for these generated files, we
# cannot use the normal `SourceRootRequest.for_target()` and we instead must determine
# a source root for every individual generated file. So, we re-resolve the codegen sources here.
python_and_resources_targets = []
codegen_targets = []
for tgt in request.targets:
if tgt.has_field(PythonSourceField) or tgt.has_field(
ResourceSourceField):
python_and_resources_targets.append(tgt)
elif tgt.get(SourcesField).can_generate(
PythonSourceField,
union_membership) or tgt.get(SourcesField).can_generate(
ResourceSourceField, union_membership):
codegen_targets.append(tgt)
codegen_sources = await MultiGet(
Get(
HydratedSources,
HydrateSourcesRequest(
tgt.get(SourcesField),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
) for tgt in codegen_targets)
source_root_requests = [
*(SourceRootRequest.for_target(tgt)
for tgt in python_and_resources_targets),
*(SourceRootRequest.for_file(f) for sources in codegen_sources
for f in sources.snapshot.files),
]
source_root_objs = await MultiGet(
Get(SourceRoot, SourceRootRequest, req)
for req in source_root_requests)
source_root_paths = {
source_root_obj.path
for source_root_obj in source_root_objs
}
return PythonSourceFiles(
SourceFiles(init_injected, sources.unrooted_files),
tuple(sorted(source_root_paths)))
def empty(cls) -> PythonSourceFiles:
return cls(SourceFiles(EMPTY_SNAPSHOT, tuple()), tuple())
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(Addresses([field_set.address]), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)
)
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.interpreter_constraints,
additional_args=field_set.generate_additional_args(python_binary_defaults),
internal_only=True,
),
)
requirements, sources, runner_pex = await MultiGet(
requirements_request, sources_request, runner_pex_request
)
merged_digest = await Get(
Digest,
MergeDigests(
[requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest]
),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict,
"PEX_PATH": in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
}
return RunRequest(
digest=merged_digest,
args=(in_chroot(runner_pex.name), "-m", entry_point),
extra_env=extra_env,
)
def run_lint_rule(
rule_runner: RuleRunner,
*,
lint_request_types: Sequence[Type[LintTargetsRequest]],
fmt_request_types: Sequence[Type[FmtRequest]] = (),
targets: list[Target],
run_files_linter: bool = False,
batch_size: int = 128,
only: list[str] | None = None,
skip_formatters: bool = False,
) -> Tuple[int, str]:
union_membership = UnionMembership({
LintTargetsRequest:
lint_request_types,
LintFilesRequest: [MockFilesRequest] if run_files_linter else [],
FmtRequest:
fmt_request_types,
})
lint_subsystem = create_goal_subsystem(
LintSubsystem,
batch_size=batch_size,
only=only or [],
skip_formatters=skip_formatters,
)
with mock_console(rule_runner.options_bootstrapper) as (console,
stdio_reader):
result: Lint = run_rule_with_mocks(
lint,
rule_args=[
console,
Workspace(rule_runner.scheduler, _enforce_effects=False),
Specs.empty(),
lint_subsystem,
union_membership,
DistDir(relpath=Path("dist")),
],
mock_gets=[
MockGet(
output_type=SourceFiles,
input_type=SourceFilesRequest,
mock=lambda _: SourceFiles(EMPTY_SNAPSHOT, ()),
),
MockGet(
output_type=LintResults,
input_type=LintTargetsRequest,
mock=lambda mock_request: mock_request.lint_results,
),
MockGet(
output_type=LintResults,
input_type=LintFilesRequest,
mock=lambda mock_request: mock_request.lint_results,
),
MockGet(
output_type=FmtResult,
input_type=FmtRequest,
mock=lambda mock_request: mock_request.fmt_result,
),
MockGet(
output_type=FilteredTargets,
input_type=Specs,
mock=lambda _: FilteredTargets(targets),
),
MockGet(
output_type=SpecsPaths,
input_type=Specs,
mock=lambda _: SpecsPaths(("f.txt", ), ()),
),
],
union_membership=union_membership,
)
assert not stdio_reader.get_stdout()
return result.exit_code, stdio_reader.get_stderr()