async def run_python_test(
field_set: PythonTestFieldSet,
test_setup: TestTargetSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
global_options: GlobalOptions,
test_options: TestOptions,
) -> TestResult:
"""Runs pytest for one target."""
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
if test_setup.xml_dir:
test_results_file = f"{field_set.address.path_safe_spec}.xml"
add_opts.extend(
(f"--junitxml={test_results_file}", f"-o junit_family={test_setup.junit_family}",)
)
env = {"PYTEST_ADDOPTS": " ".join(add_opts)}
use_coverage = test_options.values.use_coverage
output_dirs = [".coverage"] if use_coverage else []
if test_setup.xml_dir:
output_dirs.append(test_results_file)
process = test_setup.test_runner_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{test_setup.test_runner_pex.output_filename}",
pex_args=test_setup.args,
input_digest=test_setup.input_digest,
output_directories=tuple(output_dirs) if output_dirs else None,
description=f"Run Pytest for {field_set.address.reference()}",
timeout_seconds=test_setup.timeout_seconds,
env=env,
)
result = await Get[FallibleProcessResult](Process, process)
coverage_data = None
if use_coverage:
coverage_snapshot = await Get[Snapshot](
SnapshotSubset(result.output_digest, PathGlobs([".coverage"]))
)
coverage_data = PytestCoverageData(field_set.address, coverage_snapshot.digest)
xml_results_digest = None
if test_setup.xml_dir:
xml_results_snapshot = await Get[Snapshot](
SnapshotSubset(result.output_digest, PathGlobs([test_results_file]))
)
xml_results_digest = await Get[Digest](
AddPrefix(xml_results_snapshot.digest, test_setup.xml_dir)
)
return TestResult.from_fallible_process_result(
result, coverage_data=coverage_data, xml_results=xml_results_digest
)
async def get_sources(request: SetupPySourcesRequest,
source_root_config: SourceRootConfig) -> SetupPySources:
targets = request.hydrated_targets
stripped_srcs_list = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, target)
for target in targets)
# Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
# for imports to work. Note that if a repo has multiple exported targets under a single ancestor
# package, then that package must be a namespace package, which in Python 3 means it must not
# have an __init__.py. We don't validate this here, because it would require inspecting *all*
# targets, whether or not they are in the target set for this run - basically the entire repo.
# So it's the repo owners' responsibility to ensure __init__.py hygiene.
stripped_srcs_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in stripped_srcs_list
]
ancestor_init_pys = await Get[AncestorInitPyFiles](HydratedTargets,
targets)
sources_digest = await Get[Digest](DirectoriesToMerge(directories=tuple(
[*stripped_srcs_digests, *ancestor_init_pys.digests])))
init_pys_snapshot = await Get[Snapshot](SnapshotSubset(
sources_digest, PathGlobs(['**/__init__.py'])))
init_py_contents = await Get[FilesContent](
Digest, init_pys_snapshot.directory_digest)
packages, namespace_packages, package_data = find_packages(
source_roots=source_root_config.get_source_roots(),
tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
init_py_contents=init_py_contents,
py2=request.py2)
return SetupPySources(digest=sources_digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data)
def test_snapshot_subset_globs(self) -> None:
ss = SnapshotSubset(
directory_digest=self.generate_original_digest(),
globs=PathGlobs(("a.txt", "c.txt", "subdir2/**")),
)
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(subset_snapshot.files) == {
"a.txt",
"c.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
}
assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"}
content = b"dummy content"
subset_input = InputFilesContent(
(
FileContent(path="a.txt", content=content),
FileContent(path="c.txt", content=content),
FileContent(path="subdir2/a.txt", content=content),
FileContent(path="subdir2/nested_subdir/x.txt", content=content),
)
)
subset_digest = self.request_single_product(Digest, subset_input)
assert subset_snapshot.directory_digest == subset_digest
def test_snapshot_subset_globs_2(self) -> None:
ss = SnapshotSubset(self.generate_original_digest(),
PathGlobs(("a.txt", "c.txt", "subdir2/*")))
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(
subset_snapshot.files) == {"a.txt", "c.txt", "subdir2/a.txt"}
assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"}
def test_empty_snapshot_subset(self) -> None:
ss = SnapshotSubset(
directory_digest=self.generate_original_digest(),
globs=PathGlobs(()),
)
subset_snapshot = self.request_single_product(Snapshot, ss)
assert subset_snapshot.directory_digest == EMPTY_DIRECTORY_DIGEST
assert subset_snapshot.files == ()
assert subset_snapshot.dirs == ()
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest, ) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
if not request.snapshot.files:
return SourceRootStrippedSources(request.snapshot, FrozenDict())
if request.representative_path is not None:
source_root_obj = await Get[SourceRoot](
SourceRootRequest,
SourceRootRequest.for_file(request.representative_path))
source_root = source_root_obj.path
if source_root == ".":
return SourceRootStrippedSources.for_single_source_root(
request.snapshot, source_root)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources.for_single_source_root(
resulting_snapshot, source_root)
source_roots = await MultiGet(
Get[SourceRoot](SourceRootRequest, SourceRootRequest.for_file(file))
for file in request.snapshot.files)
file_to_source_root = dict(zip(request.snapshot.files, source_roots))
files_grouped_by_source_root = {
source_root.path: tuple(files)
for source_root, files in itertools.groupby(
request.snapshot.files, key=file_to_source_root.__getitem__)
}
if len(files_grouped_by_source_root) == 1:
source_root = next(iter(files_grouped_by_source_root.keys()))
if source_root == ".":
return SourceRootStrippedSources.for_single_source_root(
request.snapshot, source_root)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources.for_single_source_root(
resulting_snapshot, source_root)
snapshot_subsets = await MultiGet(
Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs(
files))) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](RemovePrefix(snapshot.digest, source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests))
return SourceRootStrippedSources(
resulting_snapshot,
FrozenDict({
source_root: tuple(file[len(source_root) + 1:] for file in files)
for source_root, files in files_grouped_by_source_root.items()
}),
)
def test_snapshot_subset_globs_2(self) -> None:
ss = SnapshotSubset(
directory_digest=self.generate_original_digest(),
globs=PathGlobs(("a.txt", "c.txt", "subdir2/*")),
)
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(
subset_snapshot.files) == {'a.txt', 'c.txt', 'subdir2/a.txt'}
assert set(subset_snapshot.dirs) == {'subdir2/nested_subdir'}
def calculate_specified_sources(
sources_snapshot: Snapshot,
origin: OriginSpec) -> Union[Snapshot, SnapshotSubset]:
# AddressSpecs simply use the entire `sources` field.
if isinstance(origin, AddressSpec):
return sources_snapshot
# NB: we ensure that `precise_files_specified` is a subset of the original `sources` field.
# It's possible when given a glob filesystem spec that the spec will have
# resolved files not belonging to this target - those must be filtered out.
precise_files_specified = set(sources_snapshot.files).intersection(
origin.resolved_files)
return SnapshotSubset(sources_snapshot.digest,
PathGlobs(sorted(precise_files_specified)))
def test_nonexistent_filename_globs(self) -> None:
# We expect to ignore, rather than error, on files that don't exist in the original snapshot.
ss = SnapshotSubset(directory_digest=self.generate_original_digest(),
globs=PathGlobs(
("some_file_not_in_snapshot.txt", "a.txt")))
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(subset_snapshot.files) == {"a.txt"}
content = b'dummy content'
subset_input = InputFilesContent((FileContent(path='a.txt',
content=content), ))
subset_digest = self.request_single_product(Digest, subset_input)
assert subset_snapshot.directory_digest == subset_digest
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
if not request.snapshot.files:
return SourceRootStrippedSources(request.snapshot)
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
return cast(str, source_roots_object.strict_find_by_path(path).path)
if request.representative_path is not None:
source_root = determine_source_root(request.representative_path)
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
if len(files_grouped_by_source_root) == 1:
source_root = next(iter(files_grouped_by_source_root.keys()))
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
snapshot_subsets = await MultiGet(
Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs(
files))) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](RemovePrefix(snapshot.digest, source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests))
return SourceRootStrippedSources(resulting_snapshot)
def determine_specified_sources_for_target(
adaptor_with_origin: TargetAdaptorWithOrigin,
) -> Union[Snapshot, SnapshotSubset]:
adaptor = adaptor_with_origin.adaptor
origin = adaptor_with_origin.origin
sources_snapshot = cast(Snapshot, adaptor.sources.snapshot)
# AddressSpecs simply use the entire `sources` field.
if isinstance(origin, AddressSpec):
return sources_snapshot
# NB: we ensure that `precise_files_specified` is a subset of the original target's
# `sources`. It's possible when given a glob filesystem spec that the spec will have
# resolved files not belonging to this target - those must be filtered out.
precise_files_specified = set(sources_snapshot.files).intersection(origin.resolved_files)
return SnapshotSubset(
directory_digest=sources_snapshot.directory_digest,
globs=PathGlobs(sorted(precise_files_specified)),
)
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
source_root = source_roots_object.safe_find_by_path(path)
if source_root is not None:
return cast(str, source_root.path)
if source_root_config.options.unmatched == "fail":
raise NoSourceRootError(
f"Could not find a source root for `{path}`.")
# Otherwise, create a source root by using the parent directory.
return PurePath(path).parent.as_posix()
if request.representative_path is not None:
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=request.snapshot.directory_digest,
prefix=determine_source_root(request.representative_path),
))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset(
directory_digest=request.snapshot.directory_digest,
globs=PathGlobs(files),
)) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests))
resulting_snapshot = await Get[Snapshot](Digest, merged_result)
return SourceRootStrippedSources(resulting_snapshot)
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources:
targets = request.targets
stripped_srcs_list = await MultiGet(
Get[SourceRootStrippedSources](StripSourcesFieldRequest(
target.get(Sources),
for_sources_types=(PythonSources, ResourcesSources),
enable_codegen=True,
)) for target in targets)
# Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
# for imports to work. Note that if a repo has multiple exported targets under a single ancestor
# package, then that package must be a namespace package, which in Python 3 means it must not
# have an __init__.py. We don't validate this here, because it would require inspecting *all*
# targets, whether or not they are in the target set for this run - basically the entire repo.
# So it's the repo owners' responsibility to ensure __init__.py hygiene.
stripped_srcs_digests = [
stripped_sources.snapshot.digest
for stripped_sources in stripped_srcs_list
]
ancestor_init_pys = await Get[AncestorInitPyFiles](Targets, targets)
sources_digest = await Get[Digest](MergeDigests(
(*stripped_srcs_digests, *ancestor_init_pys.digests)))
init_pys_snapshot = await Get[Snapshot](SnapshotSubset(
sources_digest, PathGlobs(["**/__init__.py"])))
init_py_contents = await Get[FilesContent](Digest,
init_pys_snapshot.digest)
packages, namespace_packages, package_data = find_packages(
tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
init_py_contents=init_py_contents,
py2=request.py2,
)
return SetupPySources(
digest=sources_digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data,
)
