def test_strip_snapshot(rule_runner: RuleRunner) -> None: def get_stripped_files_for_snapshot( paths: List[str], *, source_root_patterns: Sequence[str] = ("src/python", "src/java", "tests/python"), ) -> List[str]: input_snapshot = rule_runner.make_snapshot_of_empty_files(paths) request = SourceFiles(input_snapshot, ()) return get_stripped_files(rule_runner, request, source_root_patterns=source_root_patterns) # Normal source roots assert get_stripped_files_for_snapshot(["src/python/project/example.py" ]) == ["project/example.py"] assert (get_stripped_files_for_snapshot( ["src/python/project/example.py"], ) == ["project/example.py"]) assert get_stripped_files_for_snapshot( ["src/java/com/project/example.java"]) == ["com/project/example.java"] assert get_stripped_files_for_snapshot([ "tests/python/project_test/example.py" ]) == ["project_test/example.py"] # Unrecognized source root unrecognized_source_root = "no-source-root/example.txt" with pytest.raises(ExecutionError) as exc: get_stripped_files_for_snapshot([unrecognized_source_root]) assert f"NoSourceRootError: No source root found for `{unrecognized_source_root}`." in str( exc.value) # Support for multiple source roots file_names = [ "src/python/project/example.py", "src/java/com/project/example.java" ] assert get_stripped_files_for_snapshot(file_names) == [ "com/project/example.java", "project/example.py", ] # Test a source root at the repo root. We have performance optimizations for this case # because there is nothing to strip. assert get_stripped_files_for_snapshot(["project/f1.py", "project/f2.py"], source_root_patterns=["/"]) == [ "project/f1.py", "project/f2.py" ] assert get_stripped_files_for_snapshot(["dir1/f.py", "dir2/f.py"], source_root_patterns=["/"]) == [ "dir1/f.py", "dir2/f.py" ] # Gracefully handle an empty snapshot assert get_stripped_files(rule_runner, SourceFiles(EMPTY_SNAPSHOT, ())) == []
def test_build_local_dists(rule_runner: RuleRunner) -> None: foo = PurePath("foo") rule_runner.write_files({ foo / "BUILD": dedent(""" python_sources() python_distribution( name = "dist", dependencies = [":foo"], provides = python_artifact(name="foo", version="9.8.7"), sdist = False, generate_setup = False, ) """), foo / "bar.py": "BAR = 42", foo / "setup.py": dedent(""" from setuptools import setup setup(name="foo", version="9.8.7", packages=["foo"], package_dir={"foo": "."},) """), }) rule_runner.set_options([], env_inherit={"PATH"}) sources_digest = rule_runner.request( Digest, [ CreateDigest([ FileContent("srcroot/foo/bar.py", b""), FileContent("srcroot/foo/qux.py", b"") ]) ], ) sources_snapshot = rule_runner.request(Snapshot, [sources_digest]) sources = PythonSourceFiles(SourceFiles(sources_snapshot, tuple()), ("srcroot", )) request = LocalDistsPexRequest([Address("foo", target_name="dist")], internal_only=True, sources=sources) result = rule_runner.request(LocalDistsPex, [request]) assert result.pex is not None contents = rule_runner.request(DigestContents, [result.pex.digest]) whl_content = None for content in contents: if content.path == "local_dists.pex/.deps/foo-9.8.7-py3-none-any.whl": whl_content = content assert whl_content with io.BytesIO(whl_content.content) as fp: with zipfile.ZipFile(fp, "r") as whl: assert "foo/bar.py" in whl.namelist() # Check that srcroot/foo/bar.py was subtracted out, because the dist provides foo/bar.py. assert result.remaining_sources.source_files.files == ( "srcroot/foo/qux.py", )
def get_stripped_files_for_snapshot( paths: list[str], *, source_root_patterns: Sequence[str] = ("src/python", "src/java", "tests/python"), ) -> list[str]: input_snapshot = rule_runner.make_snapshot_of_empty_files(paths) request = SourceFiles(input_snapshot, ()) return get_stripped_files(rule_runner, request, source_root_patterns=source_root_patterns)
def __init__( self, addresses: Iterable[Address], *, interpreter_constraints: InterpreterConstraints = InterpreterConstraints(), sources: PythonSourceFiles = PythonSourceFiles( SourceFiles(EMPTY_SNAPSHOT, tuple()), tuple()), ) -> None: self.addresses = Addresses(addresses) self.interpreter_constraints = interpreter_constraints self.sources = sources
async def prepare_python_sources( request: PythonSourceFilesRequest, union_membership: UnionMembership) -> PythonSourceFiles: sources = await Get( SourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in request.targets), for_sources_types=request.valid_sources_types, enable_codegen=True, ), ) missing_init_files = await Get( AncestorFiles, AncestorFilesRequest("__init__.py", sources.snapshot), ) init_injected = await Get( Snapshot, MergeDigests( (sources.snapshot.digest, missing_init_files.snapshot.digest)), ) source_root_objs = await MultiGet( Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(tgt)) for tgt in request.targets if (tgt.has_field(PythonSources) or tgt.has_field(ResourcesSources) or tgt.get(Sources).can_generate(PythonSources, union_membership) or tgt.get(Sources).can_generate(ResourcesSources, union_membership))) source_root_paths = { source_root_obj.path for source_root_obj in source_root_objs } return PythonSourceFiles( SourceFiles(init_injected, sources.unrooted_files), tuple(sorted(source_root_paths)))
async def build_local_dists( request: LocalDistsPexRequest, ) -> LocalDistsPex: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) applicable_targets = [ tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt) ] local_dists_wheels = await MultiGet( Get(LocalDistWheels, PythonDistributionFieldSet, PythonDistributionFieldSet.create(target)) for target in applicable_targets ) # The primary use-case of the "local dists" feature is to support consuming native extensions # as wheels without having to publish them first. # It doesn't seem very useful to consume locally-built sdists, and it makes it hard to # reason about possible sys.path collisions between the in-repo sources and whatever the # sdist will place on the sys.path when it's installed. # So for now we simply ignore sdists, with a warning if necessary. provided_files: set[str] = set() wheels: list[str] = [] wheels_digests = [] for local_dist_wheels in local_dists_wheels: wheels.extend(local_dist_wheels.wheel_paths) wheels_digests.append(local_dist_wheels.wheels_digest) provided_files.update(local_dist_wheels.provided_files) wheels_digest = await Get(Digest, MergeDigests(wheels_digests)) dists_pex = await Get( Pex, PexRequest( output_filename="local_dists.pex", requirements=PexRequirements(wheels), interpreter_constraints=request.interpreter_constraints, additional_inputs=wheels_digest, internal_only=request.internal_only, additional_args=["--intransitive"], ), ) if not wheels: # The source calculations below are not (always) cheap, so we skip them if no wheels were # produced. See https://github.com/pantsbuild/pants/issues/14561 for one possible approach # to sharing the cost of these calculations. return LocalDistsPex(dists_pex, request.sources) # We check source roots in reverse lexicographic order, # so we'll find the innermost root that matches. source_roots = sorted(request.sources.source_roots, reverse=True) remaining_sources = set(request.sources.source_files.files) unrooted_files_set = set(request.sources.source_files.unrooted_files) for source in request.sources.source_files.files: if source not in unrooted_files_set: for source_root in source_roots: source_relpath = fast_relpath_optional(source, source_root) if source_relpath is not None and source_relpath in provided_files: remaining_sources.remove(source) remaining_sources_snapshot = await Get( Snapshot, DigestSubset( request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources)) ), ) subtracted_sources = PythonSourceFiles( SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files), request.sources.source_roots, ) return LocalDistsPex(dists_pex, subtracted_sources)
async def build_local_dists(request: LocalDistsPexRequest, ) -> LocalDistsPex: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) applicable_targets = [ tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt) ] python_dist_field_sets = [ PythonDistributionFieldSet.create(target) for target in applicable_targets ] dists = await MultiGet([ Get(BuiltPackage, PackageFieldSet, field_set) for field_set in python_dist_field_sets ]) # The primary use-case of the "local dists" feature is to support consuming native extensions # as wheels without having to publish them first. # It doesn't seem very useful to consume locally-built sdists, and it makes it hard to # reason about possible sys.path collisions between the in-repo sources and whatever the # sdist will place on the sys.path when it's installed. # So for now we simply ignore sdists, with a warning if necessary. provided_files = set() wheels = [] all_contents = await MultiGet( Get(DigestContents, Digest, dist.digest) for dist in dists) for dist, contents, tgt in zip(dists, all_contents, applicable_targets): artifacts = set((a.relpath or "") for a in dist.artifacts) # A given local dist might build a wheel and an sdist (and maybe other artifacts - # we don't know what setup command was run...) # As long as there is a wheel, we can ignore the other artifacts. wheel = next((art for art in artifacts if art.endswith(".whl")), None) if wheel: wheel_content = next(content for content in contents if content.path == wheel) wheels.append(wheel) buf = BytesIO() buf.write(wheel_content.content) buf.seek(0) with zipfile.ZipFile(buf) as zf: provided_files.update(zf.namelist()) else: logger.warning( f"Encountered a dependency on the {tgt.alias} target at {tgt.address.spec}, but " "this target does not produce a Python wheel artifact. Therefore this target's " "code will be used directly from sources, without a distribution being built, " "and therefore any native extensions in it will not be built.\n\n" f"See {doc_url('python-distributions')} for details on how to set up a {tgt.alias} " "target to produce a wheel.") dists_digest = await Get(Digest, MergeDigests([dist.digest for dist in dists])) wheels_digest = await Get( Digest, DigestSubset(dists_digest, PathGlobs(["**/*.whl"]))) dists_pex = await Get( Pex, PexRequest( output_filename="local_dists.pex", requirements=PexRequirements(wheels), interpreter_constraints=request.interpreter_constraints, additional_inputs=wheels_digest, internal_only=True, ), ) # We check source roots in reverse lexicographic order, # so we'll find the innermost root that matches. source_roots = list(reversed(sorted(request.sources.source_roots))) remaining_sources = set(request.sources.source_files.files) unrooted_files_set = set(request.sources.source_files.unrooted_files) for source in request.sources.source_files.files: if source not in unrooted_files_set: for source_root in source_roots: if (source.startswith(source_root) and os.path.relpath( source, source_root) in provided_files): remaining_sources.remove(source) remaining_sources_snapshot = await Get( Snapshot, DigestSubset(request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources))), ) subtracted_sources = PythonSourceFiles( SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files), request.sources.source_roots, ) return LocalDistsPex(dists_pex, subtracted_sources)
async def prepare_python_sources( request: PythonSourceFilesRequest, union_membership: UnionMembership) -> PythonSourceFiles: sources = await Get( SourceFiles, SourceFilesRequest( (tgt.get(SourcesField) for tgt in request.targets), for_sources_types=request.valid_sources_types, enable_codegen=True, ), ) missing_init_files = await Get( AncestorFiles, AncestorFilesRequest(input_files=sources.snapshot.files, requested=("__init__.py", "__init__.pyi")), ) init_injected = await Get( Snapshot, MergeDigests( (sources.snapshot.digest, missing_init_files.snapshot.digest))) # Codegen is able to generate code in any arbitrary location, unlike sources normally being # rooted under the target definition. To determine source roots for these generated files, we # cannot use the normal `SourceRootRequest.for_target()` and we instead must determine # a source root for every individual generated file. So, we re-resolve the codegen sources here. python_and_resources_targets = [] codegen_targets = [] for tgt in request.targets: if tgt.has_field(PythonSourceField) or tgt.has_field( ResourceSourceField): python_and_resources_targets.append(tgt) elif tgt.get(SourcesField).can_generate( PythonSourceField, union_membership) or tgt.get(SourcesField).can_generate( ResourceSourceField, union_membership): codegen_targets.append(tgt) codegen_sources = await MultiGet( Get( HydratedSources, HydrateSourcesRequest( tgt.get(SourcesField), for_sources_types=request.valid_sources_types, enable_codegen=True, ), ) for tgt in codegen_targets) source_root_requests = [ *(SourceRootRequest.for_target(tgt) for tgt in python_and_resources_targets), *(SourceRootRequest.for_file(f) for sources in codegen_sources for f in sources.snapshot.files), ] source_root_objs = await MultiGet( Get(SourceRoot, SourceRootRequest, req) for req in source_root_requests) source_root_paths = { source_root_obj.path for source_root_obj in source_root_objs } return PythonSourceFiles( SourceFiles(init_injected, sources.unrooted_files), tuple(sorted(source_root_paths)))
def empty(cls) -> PythonSourceFiles: return cls(SourceFiles(EMPTY_SNAPSHOT, tuple()), tuple())
async def create_python_binary_run_request( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point = field_set.entry_point.value if entry_point is None: # TODO: This is overkill? We don't need to hydrate the sources and strip snapshots, # we only need the path relative to the source root. binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources)) stripped_binary_sources = await Get( StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()) ) entry_point = PythonBinarySources.translate_source_file_to_entry_point( stripped_binary_sources.snapshot.files ) if entry_point is None: raise InvalidFieldException( "You must either specify `sources` or `entry_point` for the target " f"{repr(field_set.address)} in order to run it, but both fields were undefined." ) transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address])) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements(Addresses([field_set.address]), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True) ) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request.interpreter_constraints, additional_args=field_set.generate_additional_args(python_binary_defaults), internal_only=True, ), ) requirements, sources, runner_pex = await MultiGet( requirements_request, sources_request, runner_pex_request ) merged_digest = await Get( Digest, MergeDigests( [requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest] ), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict, "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest( digest=merged_digest, args=(in_chroot(runner_pex.name), "-m", entry_point), extra_env=extra_env, )
def run_lint_rule( rule_runner: RuleRunner, *, lint_request_types: Sequence[Type[LintTargetsRequest]], fmt_request_types: Sequence[Type[FmtRequest]] = (), targets: list[Target], run_files_linter: bool = False, batch_size: int = 128, only: list[str] | None = None, skip_formatters: bool = False, ) -> Tuple[int, str]: union_membership = UnionMembership({ LintTargetsRequest: lint_request_types, LintFilesRequest: [MockFilesRequest] if run_files_linter else [], FmtRequest: fmt_request_types, }) lint_subsystem = create_goal_subsystem( LintSubsystem, batch_size=batch_size, only=only or [], skip_formatters=skip_formatters, ) with mock_console(rule_runner.options_bootstrapper) as (console, stdio_reader): result: Lint = run_rule_with_mocks( lint, rule_args=[ console, Workspace(rule_runner.scheduler, _enforce_effects=False), Specs.empty(), lint_subsystem, union_membership, DistDir(relpath=Path("dist")), ], mock_gets=[ MockGet( output_type=SourceFiles, input_type=SourceFilesRequest, mock=lambda _: SourceFiles(EMPTY_SNAPSHOT, ()), ), MockGet( output_type=LintResults, input_type=LintTargetsRequest, mock=lambda mock_request: mock_request.lint_results, ), MockGet( output_type=LintResults, input_type=LintFilesRequest, mock=lambda mock_request: mock_request.lint_results, ), MockGet( output_type=FmtResult, input_type=FmtRequest, mock=lambda mock_request: mock_request.fmt_result, ), MockGet( output_type=FilteredTargets, input_type=Specs, mock=lambda _: FilteredTargets(targets), ), MockGet( output_type=SpecsPaths, input_type=Specs, mock=lambda _: SpecsPaths(("f.txt", ), ()), ), ], union_membership=union_membership, ) assert not stdio_reader.get_stdout() return result.exit_code, stdio_reader.get_stderr()