async def determine_specified_source_files(
request: SpecifiedSourceFilesRequest) -> SourceFiles:
"""Determine the specified `sources` for targets, possibly finding a subset of the original
`sources` fields if the user supplied file arguments."""
all_hydrated_sources = await MultiGet(
Get[HydratedSources](HydrateSourcesRequest(
sources_field_with_origin[0],
for_sources_types=request.for_sources_types,
enable_codegen=request.enable_codegen,
))
for sources_field_with_origin in request.sources_fields_with_origins)
full_snapshots = {}
snapshot_subset_requests = {}
for hydrated_sources, sources_field_with_origin in zip(
all_hydrated_sources, request.sources_fields_with_origins):
sources_field, origin = sources_field_with_origin
if not hydrated_sources.snapshot.files:
continue
result = calculate_specified_sources(hydrated_sources.snapshot, origin)
if isinstance(result, Snapshot):
full_snapshots[sources_field] = result
else:
snapshot_subset_requests[sources_field] = result
snapshot_subsets: Tuple[Snapshot, ...] = ()
if snapshot_subset_requests:
snapshot_subsets = await MultiGet(
Get[Snapshot](SnapshotSubset, request)
for request in snapshot_subset_requests.values())
all_snapshots: Iterable[Snapshot] = (*full_snapshots.values(),
*snapshot_subsets)
if request.strip_source_roots:
all_sources_fields = (*full_snapshots.keys(),
*snapshot_subset_requests.keys())
stripped_snapshots = await MultiGet(
Get[SourceRootStrippedSources](StripSourcesFieldRequest(
sources_field,
specified_files_snapshot=snapshot,
for_sources_types=request.for_sources_types,
enable_codegen=request.enable_codegen,
)) for sources_field, snapshot in zip(all_sources_fields,
all_snapshots))
all_snapshots = (stripped_snapshot.snapshot
for stripped_snapshot in stripped_snapshots)
result = await Get[Snapshot](MergeDigests(snapshot.digest
for snapshot in all_snapshots))
return SourceFiles(result)
async def determine_all_source_files(
request: AllSourceFilesRequest) -> SourceFiles:
"""Merge all `Sources` fields into one Snapshot."""
if request.strip_source_roots:
stripped_snapshots = await MultiGet(
Get[SourceRootStrippedSources](StripSourcesFieldRequest(
sources_field)) for sources_field in request.sources_fields)
digests_to_merge = tuple(stripped_snapshot.snapshot.directory_digest
for stripped_snapshot in stripped_snapshots)
else:
all_hydrated_sources = await MultiGet(
Get[HydratedSources](HydrateSourcesRequest, sources_field.request)
for sources_field in request.sources_fields)
digests_to_merge = tuple(hydrated_sources.snapshot.directory_digest
for hydrated_sources in all_hydrated_sources)
result = await Get[Snapshot](DirectoriesToMerge(digests_to_merge))
return SourceFiles(result)
async def get_sources(
request: SetupPySourcesRequest, source_root_config: SourceRootConfig
) -> SetupPySources:
targets = request.targets
stripped_srcs_list = await MultiGet(
Get[SourceRootStrippedSources](
StripSourcesFieldRequest(
target.get(Sources),
for_sources_types=(PythonSources, ResourcesSources),
enable_codegen=True,
)
)
for target in targets
)
# Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
# for imports to work. Note that if a repo has multiple exported targets under a single ancestor
# package, then that package must be a namespace package, which in Python 3 means it must not
# have an __init__.py. We don't validate this here, because it would require inspecting *all*
# targets, whether or not they are in the target set for this run - basically the entire repo.
# So it's the repo owners' responsibility to ensure __init__.py hygiene.
stripped_srcs_digests = [
stripped_sources.snapshot.digest for stripped_sources in stripped_srcs_list
]
ancestor_init_pys = await Get[AncestorInitPyFiles](Targets, targets)
sources_digest = await Get[Digest](
MergeDigests((*stripped_srcs_digests, *ancestor_init_pys.digests))
)
init_pys_snapshot = await Get[Snapshot](
SnapshotSubset(sources_digest, PathGlobs(["**/__init__.py"]))
)
init_py_contents = await Get[FilesContent](Digest, init_pys_snapshot.digest)
packages, namespace_packages, package_data = find_packages(
source_roots=source_root_config.get_source_roots(),
tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
init_py_contents=init_py_contents,
py2=request.py2,
)
return SetupPySources(
digest=sources_digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data,
)
def get_stripped_files_for_sources_field(
*,
source_files: Optional[List[str]],
sources_field_cls: Type[SourcesField] = SourcesField,
specified_source_files: Optional[List[str]] = None,
) -> StrippedResponseData:
if source_files:
self.create_files(path=source_root, files=source_files)
sources_field = sources_field_cls(
source_files, address=Address.parse(f"{source_root}:lib"))
specified_sources_snapshot = (None
if not specified_source_files else
self.make_snapshot_of_empty_files(
f"{source_root}/{f}"
for f in specified_source_files))
return self.get_stripped_files(
StripSourcesFieldRequest(
sources_field,
specified_files_snapshot=specified_sources_snapshot,
))
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owner_per_import = await MultiGet(
Get[PythonModuleOwner](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
return InferredDependencies(
owner.address for owner in owner_per_import
if owner.address and owner.address != request.sources_field.address)
async def map_python_module_to_targets(
module: PythonModule,
source_roots: AllSourceRoots,
third_party_mapping: ThirdPartyModuleToAddressMapping,
) -> PythonModuleOwners:
third_party_address = third_party_mapping.address_for_module(module.module)
if third_party_address:
return PythonModuleOwners([third_party_address])
unfiltered_candidate_targets = await Get[Targets](AddressSpecs(
module.address_spec(source_root=src_root.path)
for src_root in source_roots))
candidate_targets = tuple(tgt for tgt in unfiltered_candidate_targets
if tgt.has_field(PythonSources))
sources_per_target = await MultiGet(Get[SourceRootStrippedSources](
StripSourcesFieldRequest(tgt[PythonSources]))
for tgt in candidate_targets)
candidate_files = {str(p) for p in module.possible_stripped_paths()}
return PythonModuleOwners(
tgt.address
for tgt, sources in zip(candidate_targets, sources_per_target)
if bool(candidate_files.intersection(sources.snapshot.files)))
async def map_first_party_modules_to_addresses(
) -> FirstPartyModuleToAddressMapping:
all_targets = await Get[Targets](AddressSpecs([DescendantAddresses("")]))
candidate_targets = tuple(tgt for tgt in all_targets
if tgt.has_field(PythonSources))
sources_per_target = await MultiGet(Get[SourceRootStrippedSources](
StripSourcesFieldRequest(tgt[PythonSources]))
for tgt in candidate_targets)
modules_to_addresses: Dict[str, Address] = {}
modules_with_multiple_owners: Set[str] = set()
for tgt, sources in zip(candidate_targets, sources_per_target):
for f in sources.snapshot.files:
module = PythonModule.create_from_stripped_path(PurePath(f)).module
if module in modules_to_addresses:
modules_with_multiple_owners.add(module)
else:
modules_to_addresses[module] = tgt.address
# Remove modules with ambiguous owners.
for module in modules_with_multiple_owners:
modules_to_addresses.pop(module)
return FirstPartyModuleToAddressMapping(
FrozenDict(sorted(modules_to_addresses.items())))
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owners_per_import = await MultiGet(
Get[PythonModuleOwners](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
# We conservatively only use dep inference if there is exactly one owner for an import.
return InferredDependencies(
itertools.chain.from_iterable(
owners for owners in owners_per_import if len(owners) == 1
and tuple(owners)[0] != request.sources_field.address))
async def determine_all_source_files(
request: AllSourceFilesRequest) -> SourceFiles:
"""Merge all `Sources` fields into one Snapshot."""
if request.strip_source_roots:
stripped_snapshots = await MultiGet(
Get[SourceRootStrippedSources](StripSourcesFieldRequest(
sources_field,
for_sources_types=request.for_sources_types,
enable_codegen=request.enable_codegen,
)) for sources_field in request.sources_fields)
digests_to_merge = tuple(stripped_snapshot.snapshot.digest
for stripped_snapshot in stripped_snapshots)
else:
all_hydrated_sources = await MultiGet(
Get[HydratedSources](HydrateSourcesRequest(
sources_field,
for_sources_types=request.for_sources_types,
enable_codegen=request.enable_codegen,
)) for sources_field in request.sources_fields)
digests_to_merge = tuple(hydrated_sources.snapshot.digest
for hydrated_sources in all_hydrated_sources)
result = await Get[Snapshot](MergeDigests(digests_to_merge))
return SourceFiles(result)