async def determine_specified_source_files( request: SpecifiedSourceFilesRequest) -> SourceFiles: """Determine the specified `sources` for targets, possibly finding a subset of the original `sources` fields if the user supplied file arguments.""" all_hydrated_sources = await MultiGet( Get[HydratedSources](HydrateSourcesRequest( sources_field_with_origin[0], for_sources_types=request.for_sources_types, enable_codegen=request.enable_codegen, )) for sources_field_with_origin in request.sources_fields_with_origins) full_snapshots = {} snapshot_subset_requests = {} for hydrated_sources, sources_field_with_origin in zip( all_hydrated_sources, request.sources_fields_with_origins): sources_field, origin = sources_field_with_origin if not hydrated_sources.snapshot.files: continue result = calculate_specified_sources(hydrated_sources.snapshot, origin) if isinstance(result, Snapshot): full_snapshots[sources_field] = result else: snapshot_subset_requests[sources_field] = result snapshot_subsets: Tuple[Snapshot, ...] = () if snapshot_subset_requests: snapshot_subsets = await MultiGet( Get[Snapshot](SnapshotSubset, request) for request in snapshot_subset_requests.values()) all_snapshots: Iterable[Snapshot] = (*full_snapshots.values(), *snapshot_subsets) if request.strip_source_roots: all_sources_fields = (*full_snapshots.keys(), *snapshot_subset_requests.keys()) stripped_snapshots = await MultiGet( Get[SourceRootStrippedSources](StripSourcesFieldRequest( sources_field, specified_files_snapshot=snapshot, for_sources_types=request.for_sources_types, enable_codegen=request.enable_codegen, )) for sources_field, snapshot in zip(all_sources_fields, all_snapshots)) all_snapshots = (stripped_snapshot.snapshot for stripped_snapshot in stripped_snapshots) result = await Get[Snapshot](MergeDigests(snapshot.digest for snapshot in all_snapshots)) return SourceFiles(result)
async def determine_all_source_files( request: AllSourceFilesRequest) -> SourceFiles: """Merge all `Sources` fields into one Snapshot.""" if request.strip_source_roots: stripped_snapshots = await MultiGet( Get[SourceRootStrippedSources](StripSourcesFieldRequest( sources_field)) for sources_field in request.sources_fields) digests_to_merge = tuple(stripped_snapshot.snapshot.directory_digest for stripped_snapshot in stripped_snapshots) else: all_hydrated_sources = await MultiGet( Get[HydratedSources](HydrateSourcesRequest, sources_field.request) for sources_field in request.sources_fields) digests_to_merge = tuple(hydrated_sources.snapshot.directory_digest for hydrated_sources in all_hydrated_sources) result = await Get[Snapshot](DirectoriesToMerge(digests_to_merge)) return SourceFiles(result)
async def get_sources( request: SetupPySourcesRequest, source_root_config: SourceRootConfig ) -> SetupPySources: targets = request.targets stripped_srcs_list = await MultiGet( Get[SourceRootStrippedSources]( StripSourcesFieldRequest( target.get(Sources), for_sources_types=(PythonSources, ResourcesSources), enable_codegen=True, ) ) for target in targets ) # Create a chroot with all the sources, and any ancestor __init__.py files that might be needed # for imports to work. Note that if a repo has multiple exported targets under a single ancestor # package, then that package must be a namespace package, which in Python 3 means it must not # have an __init__.py. We don't validate this here, because it would require inspecting *all* # targets, whether or not they are in the target set for this run - basically the entire repo. # So it's the repo owners' responsibility to ensure __init__.py hygiene. stripped_srcs_digests = [ stripped_sources.snapshot.digest for stripped_sources in stripped_srcs_list ] ancestor_init_pys = await Get[AncestorInitPyFiles](Targets, targets) sources_digest = await Get[Digest]( MergeDigests((*stripped_srcs_digests, *ancestor_init_pys.digests)) ) init_pys_snapshot = await Get[Snapshot]( SnapshotSubset(sources_digest, PathGlobs(["**/__init__.py"])) ) init_py_contents = await Get[FilesContent](Digest, init_pys_snapshot.digest) packages, namespace_packages, package_data = find_packages( source_roots=source_root_config.get_source_roots(), tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)), init_py_contents=init_py_contents, py2=request.py2, ) return SetupPySources( digest=sources_digest, packages=packages, namespace_packages=namespace_packages, package_data=package_data, )
def get_stripped_files_for_sources_field( *, source_files: Optional[List[str]], sources_field_cls: Type[SourcesField] = SourcesField, specified_source_files: Optional[List[str]] = None, ) -> StrippedResponseData: if source_files: self.create_files(path=source_root, files=source_files) sources_field = sources_field_cls( source_files, address=Address.parse(f"{source_root}:lib")) specified_sources_snapshot = (None if not specified_source_files else self.make_snapshot_of_empty_files( f"{source_root}/{f}" for f in specified_source_files)) return self.get_stripped_files( StripSourcesFieldRequest( sources_field, specified_files_snapshot=specified_sources_snapshot, ))
async def infer_python_dependencies( request: InferPythonDependencies) -> InferredDependencies: stripped_sources = await Get[SourceRootStrippedSources]( StripSourcesFieldRequest(request.sources_field)) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) files_content = await Get[FilesContent](Digest, stripped_sources.snapshot.digest) imports_per_file = tuple( find_python_imports(fc.content.decode(), module_name=module.module) for fc, module in zip(files_content, modules)) owner_per_import = await MultiGet( Get[PythonModuleOwner](PythonModule(imported_module)) for file_imports in imports_per_file for imported_module in file_imports.all_imports if imported_module not in combined_stdlib) return InferredDependencies( owner.address for owner in owner_per_import if owner.address and owner.address != request.sources_field.address)
async def map_python_module_to_targets( module: PythonModule, source_roots: AllSourceRoots, third_party_mapping: ThirdPartyModuleToAddressMapping, ) -> PythonModuleOwners: third_party_address = third_party_mapping.address_for_module(module.module) if third_party_address: return PythonModuleOwners([third_party_address]) unfiltered_candidate_targets = await Get[Targets](AddressSpecs( module.address_spec(source_root=src_root.path) for src_root in source_roots)) candidate_targets = tuple(tgt for tgt in unfiltered_candidate_targets if tgt.has_field(PythonSources)) sources_per_target = await MultiGet(Get[SourceRootStrippedSources]( StripSourcesFieldRequest(tgt[PythonSources])) for tgt in candidate_targets) candidate_files = {str(p) for p in module.possible_stripped_paths()} return PythonModuleOwners( tgt.address for tgt, sources in zip(candidate_targets, sources_per_target) if bool(candidate_files.intersection(sources.snapshot.files)))
async def map_first_party_modules_to_addresses( ) -> FirstPartyModuleToAddressMapping: all_targets = await Get[Targets](AddressSpecs([DescendantAddresses("")])) candidate_targets = tuple(tgt for tgt in all_targets if tgt.has_field(PythonSources)) sources_per_target = await MultiGet(Get[SourceRootStrippedSources]( StripSourcesFieldRequest(tgt[PythonSources])) for tgt in candidate_targets) modules_to_addresses: Dict[str, Address] = {} modules_with_multiple_owners: Set[str] = set() for tgt, sources in zip(candidate_targets, sources_per_target): for f in sources.snapshot.files: module = PythonModule.create_from_stripped_path(PurePath(f)).module if module in modules_to_addresses: modules_with_multiple_owners.add(module) else: modules_to_addresses[module] = tgt.address # Remove modules with ambiguous owners. for module in modules_with_multiple_owners: modules_to_addresses.pop(module) return FirstPartyModuleToAddressMapping( FrozenDict(sorted(modules_to_addresses.items())))
async def infer_python_dependencies( request: InferPythonDependencies) -> InferredDependencies: stripped_sources = await Get[SourceRootStrippedSources]( StripSourcesFieldRequest(request.sources_field)) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) files_content = await Get[FilesContent](Digest, stripped_sources.snapshot.digest) imports_per_file = tuple( find_python_imports(fc.content.decode(), module_name=module.module) for fc, module in zip(files_content, modules)) owners_per_import = await MultiGet( Get[PythonModuleOwners](PythonModule(imported_module)) for file_imports in imports_per_file for imported_module in file_imports.all_imports if imported_module not in combined_stdlib) # We conservatively only use dep inference if there is exactly one owner for an import. return InferredDependencies( itertools.chain.from_iterable( owners for owners in owners_per_import if len(owners) == 1 and tuple(owners)[0] != request.sources_field.address))
async def determine_all_source_files( request: AllSourceFilesRequest) -> SourceFiles: """Merge all `Sources` fields into one Snapshot.""" if request.strip_source_roots: stripped_snapshots = await MultiGet( Get[SourceRootStrippedSources](StripSourcesFieldRequest( sources_field, for_sources_types=request.for_sources_types, enable_codegen=request.enable_codegen, )) for sources_field in request.sources_fields) digests_to_merge = tuple(stripped_snapshot.snapshot.digest for stripped_snapshot in stripped_snapshots) else: all_hydrated_sources = await MultiGet( Get[HydratedSources](HydrateSourcesRequest( sources_field, for_sources_types=request.for_sources_types, enable_codegen=request.enable_codegen, )) for sources_field in request.sources_fields) digests_to_merge = tuple(hydrated_sources.snapshot.digest for hydrated_sources in all_hydrated_sources) result = await Get[Snapshot](MergeDigests(digests_to_merge)) return SourceFiles(result)