async def run_python_test( field_set: PythonTestFieldSet, test_setup: TestTargetSetup, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, global_options: GlobalOptions, test_options: TestOptions, ) -> TestResult: """Runs pytest for one target.""" add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] if test_setup.xml_dir: test_results_file = f"{field_set.address.path_safe_spec}.xml" add_opts.extend( (f"--junitxml={test_results_file}", f"-o junit_family={test_setup.junit_family}",) ) env = {"PYTEST_ADDOPTS": " ".join(add_opts)} use_coverage = test_options.values.use_coverage output_dirs = [".coverage"] if use_coverage else [] if test_setup.xml_dir: output_dirs.append(test_results_file) process = test_setup.test_runner_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./{test_setup.test_runner_pex.output_filename}", pex_args=test_setup.args, input_digest=test_setup.input_digest, output_directories=tuple(output_dirs) if output_dirs else None, description=f"Run Pytest for {field_set.address.reference()}", timeout_seconds=test_setup.timeout_seconds, env=env, ) result = await Get[FallibleProcessResult](Process, process) coverage_data = None if use_coverage: coverage_snapshot = await Get[Snapshot]( SnapshotSubset(result.output_digest, PathGlobs([".coverage"])) ) coverage_data = PytestCoverageData(field_set.address, coverage_snapshot.digest) xml_results_digest = None if test_setup.xml_dir: xml_results_snapshot = await Get[Snapshot]( SnapshotSubset(result.output_digest, PathGlobs([test_results_file])) ) xml_results_digest = await Get[Digest]( AddPrefix(xml_results_snapshot.digest, test_setup.xml_dir) ) return TestResult.from_fallible_process_result( result, coverage_data=coverage_data, xml_results=xml_results_digest )
async def get_sources(request: SetupPySourcesRequest, source_root_config: SourceRootConfig) -> SetupPySources: targets = request.hydrated_targets stripped_srcs_list = await MultiGet( Get[SourceRootStrippedSources](HydratedTarget, target) for target in targets) # Create a chroot with all the sources, and any ancestor __init__.py files that might be needed # for imports to work. Note that if a repo has multiple exported targets under a single ancestor # package, then that package must be a namespace package, which in Python 3 means it must not # have an __init__.py. We don't validate this here, because it would require inspecting *all* # targets, whether or not they are in the target set for this run - basically the entire repo. # So it's the repo owners' responsibility to ensure __init__.py hygiene. stripped_srcs_digests = [ stripped_sources.snapshot.directory_digest for stripped_sources in stripped_srcs_list ] ancestor_init_pys = await Get[AncestorInitPyFiles](HydratedTargets, targets) sources_digest = await Get[Digest](DirectoriesToMerge(directories=tuple( [*stripped_srcs_digests, *ancestor_init_pys.digests]))) init_pys_snapshot = await Get[Snapshot](SnapshotSubset( sources_digest, PathGlobs(['**/__init__.py']))) init_py_contents = await Get[FilesContent]( Digest, init_pys_snapshot.directory_digest) packages, namespace_packages, package_data = find_packages( source_roots=source_root_config.get_source_roots(), tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)), init_py_contents=init_py_contents, py2=request.py2) return SetupPySources(digest=sources_digest, packages=packages, namespace_packages=namespace_packages, package_data=package_data)
def test_snapshot_subset_globs(self) -> None: ss = SnapshotSubset( directory_digest=self.generate_original_digest(), globs=PathGlobs(("a.txt", "c.txt", "subdir2/**")), ) subset_snapshot = self.request_single_product(Snapshot, ss) assert set(subset_snapshot.files) == { "a.txt", "c.txt", "subdir2/a.txt", "subdir2/nested_subdir/x.txt", } assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"} content = b"dummy content" subset_input = InputFilesContent( ( FileContent(path="a.txt", content=content), FileContent(path="c.txt", content=content), FileContent(path="subdir2/a.txt", content=content), FileContent(path="subdir2/nested_subdir/x.txt", content=content), ) ) subset_digest = self.request_single_product(Digest, subset_input) assert subset_snapshot.directory_digest == subset_digest
def test_snapshot_subset_globs_2(self) -> None: ss = SnapshotSubset(self.generate_original_digest(), PathGlobs(("a.txt", "c.txt", "subdir2/*"))) subset_snapshot = self.request_single_product(Snapshot, ss) assert set( subset_snapshot.files) == {"a.txt", "c.txt", "subdir2/a.txt"} assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"}
def test_empty_snapshot_subset(self) -> None: ss = SnapshotSubset( directory_digest=self.generate_original_digest(), globs=PathGlobs(()), ) subset_snapshot = self.request_single_product(Snapshot, ss) assert subset_snapshot.directory_digest == EMPTY_DIRECTORY_DIGEST assert subset_snapshot.files == () assert subset_snapshot.dirs == ()
async def strip_source_roots_from_snapshot( request: StripSnapshotRequest, ) -> SourceRootStrippedSources: """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py`.""" if not request.snapshot.files: return SourceRootStrippedSources(request.snapshot, FrozenDict()) if request.representative_path is not None: source_root_obj = await Get[SourceRoot]( SourceRootRequest, SourceRootRequest.for_file(request.representative_path)) source_root = source_root_obj.path if source_root == ".": return SourceRootStrippedSources.for_single_source_root( request.snapshot, source_root) resulting_snapshot = await Get[Snapshot](RemovePrefix( request.snapshot.digest, source_root)) return SourceRootStrippedSources.for_single_source_root( resulting_snapshot, source_root) source_roots = await MultiGet( Get[SourceRoot](SourceRootRequest, SourceRootRequest.for_file(file)) for file in request.snapshot.files) file_to_source_root = dict(zip(request.snapshot.files, source_roots)) files_grouped_by_source_root = { source_root.path: tuple(files) for source_root, files in itertools.groupby( request.snapshot.files, key=file_to_source_root.__getitem__) } if len(files_grouped_by_source_root) == 1: source_root = next(iter(files_grouped_by_source_root.keys())) if source_root == ".": return SourceRootStrippedSources.for_single_source_root( request.snapshot, source_root) resulting_snapshot = await Get[Snapshot](RemovePrefix( request.snapshot.digest, source_root)) return SourceRootStrippedSources.for_single_source_root( resulting_snapshot, source_root) snapshot_subsets = await MultiGet( Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs( files))) for files in files_grouped_by_source_root.values()) resulting_digests = await MultiGet( Get[Digest](RemovePrefix(snapshot.digest, source_root)) for snapshot, source_root in zip(snapshot_subsets, files_grouped_by_source_root.keys())) resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests)) return SourceRootStrippedSources( resulting_snapshot, FrozenDict({ source_root: tuple(file[len(source_root) + 1:] for file in files) for source_root, files in files_grouped_by_source_root.items() }), )
def test_snapshot_subset_globs_2(self) -> None: ss = SnapshotSubset( directory_digest=self.generate_original_digest(), globs=PathGlobs(("a.txt", "c.txt", "subdir2/*")), ) subset_snapshot = self.request_single_product(Snapshot, ss) assert set( subset_snapshot.files) == {'a.txt', 'c.txt', 'subdir2/a.txt'} assert set(subset_snapshot.dirs) == {'subdir2/nested_subdir'}
def calculate_specified_sources( sources_snapshot: Snapshot, origin: OriginSpec) -> Union[Snapshot, SnapshotSubset]: # AddressSpecs simply use the entire `sources` field. if isinstance(origin, AddressSpec): return sources_snapshot # NB: we ensure that `precise_files_specified` is a subset of the original `sources` field. # It's possible when given a glob filesystem spec that the spec will have # resolved files not belonging to this target - those must be filtered out. precise_files_specified = set(sources_snapshot.files).intersection( origin.resolved_files) return SnapshotSubset(sources_snapshot.digest, PathGlobs(sorted(precise_files_specified)))
def test_nonexistent_filename_globs(self) -> None: # We expect to ignore, rather than error, on files that don't exist in the original snapshot. ss = SnapshotSubset(directory_digest=self.generate_original_digest(), globs=PathGlobs( ("some_file_not_in_snapshot.txt", "a.txt"))) subset_snapshot = self.request_single_product(Snapshot, ss) assert set(subset_snapshot.files) == {"a.txt"} content = b'dummy content' subset_input = InputFilesContent((FileContent(path='a.txt', content=content), )) subset_digest = self.request_single_product(Digest, subset_input) assert subset_snapshot.directory_digest == subset_digest
async def strip_source_roots_from_snapshot( request: StripSnapshotRequest, source_root_config: SourceRootConfig, ) -> SourceRootStrippedSources: """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py`.""" if not request.snapshot.files: return SourceRootStrippedSources(request.snapshot) source_roots_object = source_root_config.get_source_roots() def determine_source_root(path: str) -> str: return cast(str, source_roots_object.strict_find_by_path(path).path) if request.representative_path is not None: source_root = determine_source_root(request.representative_path) if source_root == ".": return SourceRootStrippedSources(request.snapshot) resulting_snapshot = await Get[Snapshot](RemovePrefix( request.snapshot.digest, source_root)) return SourceRootStrippedSources(resulting_snapshot) files_grouped_by_source_root = { source_root: tuple(files) for source_root, files in itertools.groupby(request.snapshot.files, key=determine_source_root) } if len(files_grouped_by_source_root) == 1: source_root = next(iter(files_grouped_by_source_root.keys())) if source_root == ".": return SourceRootStrippedSources(request.snapshot) resulting_snapshot = await Get[Snapshot](RemovePrefix( request.snapshot.digest, source_root)) return SourceRootStrippedSources(resulting_snapshot) snapshot_subsets = await MultiGet( Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs( files))) for files in files_grouped_by_source_root.values()) resulting_digests = await MultiGet( Get[Digest](RemovePrefix(snapshot.digest, source_root)) for snapshot, source_root in zip(snapshot_subsets, files_grouped_by_source_root.keys())) resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests)) return SourceRootStrippedSources(resulting_snapshot)
def determine_specified_sources_for_target( adaptor_with_origin: TargetAdaptorWithOrigin, ) -> Union[Snapshot, SnapshotSubset]: adaptor = adaptor_with_origin.adaptor origin = adaptor_with_origin.origin sources_snapshot = cast(Snapshot, adaptor.sources.snapshot) # AddressSpecs simply use the entire `sources` field. if isinstance(origin, AddressSpec): return sources_snapshot # NB: we ensure that `precise_files_specified` is a subset of the original target's # `sources`. It's possible when given a glob filesystem spec that the spec will have # resolved files not belonging to this target - those must be filtered out. precise_files_specified = set(sources_snapshot.files).intersection(origin.resolved_files) return SnapshotSubset( directory_digest=sources_snapshot.directory_digest, globs=PathGlobs(sorted(precise_files_specified)), )
async def strip_source_roots_from_snapshot( request: StripSnapshotRequest, source_root_config: SourceRootConfig, ) -> SourceRootStrippedSources: """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py`.""" source_roots_object = source_root_config.get_source_roots() def determine_source_root(path: str) -> str: source_root = source_roots_object.safe_find_by_path(path) if source_root is not None: return cast(str, source_root.path) if source_root_config.options.unmatched == "fail": raise NoSourceRootError( f"Could not find a source root for `{path}`.") # Otherwise, create a source root by using the parent directory. return PurePath(path).parent.as_posix() if request.representative_path is not None: resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip( directory_digest=request.snapshot.directory_digest, prefix=determine_source_root(request.representative_path), )) resulting_snapshot = await Get[Snapshot](Digest, resulting_digest) return SourceRootStrippedSources(snapshot=resulting_snapshot) files_grouped_by_source_root = { source_root: tuple(files) for source_root, files in itertools.groupby(request.snapshot.files, key=determine_source_root) } snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset( directory_digest=request.snapshot.directory_digest, globs=PathGlobs(files), )) for files in files_grouped_by_source_root.values()) resulting_digests = await MultiGet( Get[Digest](DirectoryWithPrefixToStrip( directory_digest=snapshot.directory_digest, prefix=source_root)) for snapshot, source_root in zip(snapshot_subsets, files_grouped_by_source_root.keys())) merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests)) resulting_snapshot = await Get[Snapshot](Digest, merged_result) return SourceRootStrippedSources(resulting_snapshot)
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources: targets = request.targets stripped_srcs_list = await MultiGet( Get[SourceRootStrippedSources](StripSourcesFieldRequest( target.get(Sources), for_sources_types=(PythonSources, ResourcesSources), enable_codegen=True, )) for target in targets) # Create a chroot with all the sources, and any ancestor __init__.py files that might be needed # for imports to work. Note that if a repo has multiple exported targets under a single ancestor # package, then that package must be a namespace package, which in Python 3 means it must not # have an __init__.py. We don't validate this here, because it would require inspecting *all* # targets, whether or not they are in the target set for this run - basically the entire repo. # So it's the repo owners' responsibility to ensure __init__.py hygiene. stripped_srcs_digests = [ stripped_sources.snapshot.digest for stripped_sources in stripped_srcs_list ] ancestor_init_pys = await Get[AncestorInitPyFiles](Targets, targets) sources_digest = await Get[Digest](MergeDigests( (*stripped_srcs_digests, *ancestor_init_pys.digests))) init_pys_snapshot = await Get[Snapshot](SnapshotSubset( sources_digest, PathGlobs(["**/__init__.py"]))) init_py_contents = await Get[FilesContent](Digest, init_pys_snapshot.digest) packages, namespace_packages, package_data = find_packages( tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)), init_py_contents=init_py_contents, py2=request.py2, ) return SetupPySources( digest=sources_digest, packages=packages, namespace_packages=namespace_packages, package_data=package_data, )