async def get_sources(request: SetupPySourcesRequest,
source_root_config: SourceRootConfig) -> SetupPySources:
targets = request.hydrated_targets
stripped_srcs_list = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, target)
for target in targets)
# Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
# for imports to work. Note that if a repo has multiple exported targets under a single ancestor
# package, then that package must be a namespace package, which in Python 3 means it must not
# have an __init__.py. We don't validate this here, because it would require inspecting *all*
# targets, whether or not they are in the target set for this run - basically the entire repo.
# So it's the repo owners' responsibility to ensure __init__.py hygiene.
stripped_srcs_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in stripped_srcs_list
]
ancestor_init_pys = await Get[AncestorInitPyFiles](HydratedTargets,
targets)
sources_digest = await Get[Digest](DirectoriesToMerge(directories=tuple(
[*stripped_srcs_digests, *ancestor_init_pys.digests])))
init_pys_snapshot = await Get[Snapshot](SnapshotSubset(
sources_digest, PathGlobs(['**/__init__.py'])))
init_py_contents = await Get[FilesContent](
Digest, init_pys_snapshot.directory_digest)
packages, namespace_packages, package_data = find_packages(
source_roots=source_root_config.get_source_roots(),
tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
init_py_contents=init_py_contents,
py2=request.py2)
return SetupPySources(digest=sources_digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data)
def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:
source_roots = source_root_config.get_source_roots()
all_paths: Set[str] = set()
for path in source_roots.traverse():
if path.startswith("^/"):
all_paths.add(f"{path[2:]}/")
else:
all_paths.add(f"**/{path}/")
snapshot = yield Get(Snapshot, PathGlobs(include=tuple(all_paths)))
all_source_roots: Set[SourceRoot] = set()
# The globs above can match on subdirectories of the source roots.
# For instance, `src/*/` might match 'src/rust/' as well as
# 'src/rust/engine/process_execution/bazel_protos/src/gen'.
# So we use find_by_path to verify every candidate source root.
for directory in snapshot.dirs:
match: SourceRoot = source_roots.find_by_path(directory)
if match:
all_source_roots.add(match)
yield AllSourceRoots(all_source_roots)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
if source_root is None:
# If we found no source root, use the target's dir.
# Note that when --source-unmatched is 'create' (the default) we'll never return None,
# but will return the target's dir. This check allows this code to work even if
# --source-unmatched is 'fail'.
source_root_path = target_adaptor.address.spec_path
else:
source_root_path = source_root.path
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root_path = ''
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=digest, prefix=source_root_path))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root = None
resulting_digest = await Get(
Digest,
DirectoryWithPrefixToStrip(
directory_digest=digest,
prefix=source_root.path if source_root else ""))
resulting_snapshot = await Get(Snapshot, Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def create_coverage_config(
coverage_config_request: CoverageConfigRequest, source_root_config: SourceRootConfig
) -> CoverageConfig:
sources = await Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in coverage_config_request.targets), strip_source_roots=False,
)
)
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}. This is so that coverage reports
# referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped back to the actual
# sources they reference when generating coverage reports.
def stripped_file_with_source_root(file_name: str) -> Tuple[str, str]:
source_root_object = source_roots.find_by_path(file_name)
source_root = source_root_object.path if source_root_object is not None else ""
stripped_path = file_name[len(source_root) + 1 :]
return stripped_path, source_root
stripped_files_to_source_roots = dict(
stripped_file_with_source_root(filename)
for filename in sorted(init_injected.snapshot.files)
)
default_config = dedent(
"""
[run]
branch = True
timid = False
relative_files = True
"""
)
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(default_config))
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME,
"source_to_target_base",
json.dumps(stripped_files_to_source_roots),
)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME, "test_time", json.dumps(coverage_config_request.is_test_time)
)
config_io_stream = StringIO()
config_parser.write(config_io_stream)
digest = await Get[Digest](
InputFilesContent(
[FileContent(".coveragerc", content=config_io_stream.getvalue().encode())]
)
)
return CoverageConfig(digest)
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
if not request.snapshot.files:
return SourceRootStrippedSources(request.snapshot)
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
return cast(str, source_roots_object.strict_find_by_path(path).path)
if request.representative_path is not None:
source_root = determine_source_root(request.representative_path)
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
if len(files_grouped_by_source_root) == 1:
source_root = next(iter(files_grouped_by_source_root.keys()))
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
snapshot_subsets = await MultiGet(
Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs(
files))) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](RemovePrefix(snapshot.digest, source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests))
return SourceRootStrippedSources(resulting_snapshot)
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
source_root = source_roots_object.safe_find_by_path(path)
if source_root is not None:
return cast(str, source_root.path)
if source_root_config.options.unmatched == "fail":
raise NoSourceRootError(
f"Could not find a source root for `{path}`.")
# Otherwise, create a source root by using the parent directory.
return PurePath(path).parent.as_posix()
if request.representative_path is not None:
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=request.snapshot.directory_digest,
prefix=determine_source_root(request.representative_path),
))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset(
directory_digest=request.snapshot.directory_digest,
globs=PathGlobs(files),
)) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests))
resulting_snapshot = await Get[Snapshot](Digest, merged_result)
return SourceRootStrippedSources(resulting_snapshot)
async def get_ancestor_init_py(
targets: Targets, source_root_config: SourceRootConfig
) -> AncestorInitPyFiles:
"""Find any ancestor __init__.py files for the given targets.
Includes sibling __init__.py files. Returns the files stripped of their source roots.
"""
source_roots = source_root_config.get_source_roots()
sources = await Get[SourceFiles](
AllSourceFilesRequest(tgt[PythonSources] for tgt in targets if tgt.has_field(PythonSources))
)
# Find the ancestors of all dirs containing .py files, including those dirs themselves.
source_dir_ancestors: Set[Tuple[str, str]] = set() # Items are (src_root, path incl. src_root).
for fp in sources.snapshot.files:
source_dir_ancestor = os.path.dirname(fp)
source_root = source_root_or_raise(source_roots, fp)
# Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py).
while source_dir_ancestor != source_root:
source_dir_ancestors.add((source_root, source_dir_ancestor))
source_dir_ancestor = os.path.dirname(source_dir_ancestor)
source_dir_ancestors_list = list(source_dir_ancestors) # To force a consistent order.
# Note that we must MultiGet single globs instead of a a single Get for all the globs, because
# we match each result to its originating glob (see use of zip below).
ancestor_init_py_snapshots = await MultiGet[Snapshot](
Get[Snapshot](PathGlobs, PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")]))
for source_dir_ancestor in source_dir_ancestors_list
)
source_root_stripped_ancestor_init_pys = await MultiGet[Digest](
Get[Digest](
DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_dir_ancestor[0]
)
)
for snapshot, source_dir_ancestor in zip(
ancestor_init_py_snapshots, source_dir_ancestors_list
)
)
return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
async def construct_coverage_config(
source_root_config: SourceRootConfig,
coverage_config_request: CoveragercRequest) -> Coveragerc:
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in coverage_config_request.hydrated_targets),
strip_source_roots=False,
))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}
# This is so coverage reports referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped
# back to the actual sources they reference when generating coverage reports.
def source_root_stripped_source_and_source_root(
file_name: str) -> Tuple[str, str]:
source_root = source_roots.find_by_path(file_name)
source_root_path = source_root.path if source_root is not None else ""
source_root_stripped_path = file_name[len(source_root_path) + 1:]
return (source_root_stripped_path, source_root_path)
source_to_target_base = dict(
source_root_stripped_source_and_source_root(filename)
for filename in sorted(init_injected.snapshot.files))
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(DEFAULT_COVERAGE_CONFIG))
ensure_section(config_parser, "run")
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base",
json.dumps(source_to_target_base))
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "test_time",
json.dumps(coverage_config_request.test_time))
config = StringIO()
config_parser.write(config)
coveragerc_digest = await Get[Digest](InputFilesContent,
get_coveragerc_input(
config.getvalue()))
return Coveragerc(coveragerc_digest)
async def list_backends(
backend_options: BackendsOptions,
source_roots_config: SourceRootConfig,
global_options: GlobalOptions,
console: Console,
) -> Backends:
source_roots = source_roots_config.get_source_roots()
discovered_register_pys = await Get[Snapshot](PathGlobs(
["**/*/register.py"]))
register_pys_content = await Get[FilesContent](
Digest, discovered_register_pys.digest)
backend_infos = tuple(
BackendInfo.create(fc, source_roots, global_options)
for fc in register_pys_content)
v1_backends = []
v2_backends = []
for backend in backend_infos:
if backend.is_v1:
v1_backends.append(backend)
if backend.is_v2:
v2_backends.append(backend)
with backend_options.line_oriented(console) as print_stdout:
if global_options.options.v1:
print_stdout(
format_section(v1_backends,
console,
version_number=1,
option_name="backend_packages"))
if global_options.options.v2:
print_stdout(
format_section(v2_backends,
console,
version_number=2,
option_name="backend_packages2"))
return Backends(exit_code=0)