async def infer_python_dependencies(
request: InferPythonDependencies,
python_inference: PythonInference) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies([], sibling_dependencies_inferrable=False)
stripped_sources = await Get(StrippedSourceFiles,
SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
digest_contents = await Get(DigestContents, Digest,
stripped_sources.snapshot.digest)
owner_requests: List[Get[PythonModuleOwner, PythonModule]] = []
for file_content, module in zip(digest_contents, modules):
file_imports_obj = find_python_imports(file_content.content.decode(),
module_name=module.module)
detected_imports = (file_imports_obj.all_imports
if python_inference.string_imports else
file_imports_obj.explicit_imports)
owner_requests.extend(
Get(PythonModuleOwner, PythonModule(imported_module))
for imported_module in detected_imports
if imported_module not in combined_stdlib)
owner_per_import = await MultiGet(owner_requests)
result = (
owner.address for owner in owner_per_import
if owner.address and owner.address != request.sources_field.address)
return InferredDependencies(result, sibling_dependencies_inferrable=True)
async def infer_python_dependencies(
request: InferPythonDependencies, python_inference: PythonInference
) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies()
stripped_sources = await Get(StrippedSourceFiles, SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files
)
digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(file_content.content.decode(), module_name=module.module)
for file_content, module in zip(digest_contents, modules)
)
owner_per_import = await MultiGet(
Get(PythonModuleOwner, PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.explicit_imports
if imported_module not in combined_stdlib
)
return InferredDependencies(
owner.address
for owner in owner_per_import
if (
owner.address
and owner.address.maybe_convert_to_base_target() != request.sources_field.address
)
)
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owner_per_import = await MultiGet(
Get[PythonModuleOwner](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
return InferredDependencies(
owner.address for owner in owner_per_import
if owner.address and owner.address != request.sources_field.address)
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owners_per_import = await MultiGet(
Get[PythonModuleOwners](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
# We conservatively only use dep inference if there is exactly one owner for an import.
return InferredDependencies(
itertools.chain.from_iterable(
owners for owners in owners_per_import if len(owners) == 1
and tuple(owners)[0] != request.sources_field.address))
async def infer_python_dependencies(
request: InferPythonDependencies,
python_inference: PythonInference) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies([], sibling_dependencies_inferrable=False)
stripped_sources = await Get(StrippedSourceFiles,
SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
digest_contents = await Get(DigestContents, Digest,
stripped_sources.snapshot.digest)
owners_requests: List[Get[PythonModuleOwners, PythonModule]] = []
for file_content, module in zip(digest_contents, modules):
file_imports_obj = find_python_imports(
filename=file_content.path,
content=file_content.content.decode(),
module_name=module.module,
)
detected_imports = (file_imports_obj.all_imports
if python_inference.string_imports else
file_imports_obj.explicit_imports)
owners_requests.extend(
Get(PythonModuleOwners, PythonModule(imported_module))
for imported_module in detected_imports
if imported_module not in combined_stdlib)
owners_per_import = await MultiGet(owners_requests)
# We remove the request's address so that we don't infer dependencies on self.
merged_result = sorted(
set(itertools.chain.from_iterable(owners_per_import)) -
{request.sources_field.address})
return InferredDependencies(merged_result,
sibling_dependencies_inferrable=True)
def test_create_module_from_path(stripped_path: PurePath,
expected: str) -> None:
assert PythonModule.create_from_stripped_path(
stripped_path) == PythonModule(expected)
async def find_putative_targets(
req: PutativePythonTargetsRequest,
all_owned_sources: AllOwnedSources,
python_setup: PythonSetup,
) -> PutativeTargets:
# Find library/test/test_util targets.
all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py")
all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
classified_unowned_py_files = classify_source_files(unowned_py_files)
pts = []
for tgt_type, paths in classified_unowned_py_files.items():
for dirname, filenames in group_by_dir(paths).items():
if issubclass(tgt_type, PythonTestsGeneratorTarget):
name = "tests"
kwargs = {"name": name}
elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
name = "test_utils"
kwargs = {"name": name}
else:
name = os.path.basename(dirname)
kwargs = {}
if (
python_setup.tailor_ignore_solitary_init_files
and tgt_type == PythonSourcesGeneratorTarget
and filenames == {"__init__.py"}
):
continue
pts.append(
PutativeTarget.for_target_type(
tgt_type, dirname, name, sorted(filenames), kwargs=kwargs
)
)
if python_setup.tailor_requirements_targets:
# Find requirements files.
all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt")
all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs)
unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources)
for req_file in unowned_requirements_files:
path, name = os.path.split(req_file)
pts.append(
PutativeTarget(
path=path,
# python_requirements is a macro and doesn't take a name argument, but the
# PutativeTarget still needs a name for display purposes.
name=name,
type_alias="python_requirements",
triggering_sources=[req_file],
owned_sources=[req_file],
addressable=False,
kwargs={} if name == "requirements.txt" else {"requirements_relpath": name},
)
)
if python_setup.tailor_pex_binary_targets:
# Find binary targets.
# Get all files whose content indicates that they are entry points.
digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs)
entry_points = [
file_content.path
for file_content in digest_contents
if is_entry_point(file_content.content)
]
# Get the modules for these entry points.
src_roots = await Get(
SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)
)
module_to_entry_point = {}
for entry_point in entry_points:
entry_point_path = PurePath(entry_point)
src_root = src_roots.path_to_root[entry_point_path]
stripped_entry_point = entry_point_path.relative_to(src_root.path)
module = PythonModule.create_from_stripped_path(stripped_entry_point)
module_to_entry_point[module.module] = entry_point
# Get existing binary targets for these entry points.
entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points}
possible_existing_binary_targets = await Get(
UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs)
)
possible_existing_binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField]))
for t in possible_existing_binary_targets
if t.has_field(PexEntryPointField)
)
possible_existing_entry_point_modules = {
rep.val.module for rep in possible_existing_binary_entry_points if rep.val
}
unowned_entry_point_modules = (
module_to_entry_point.keys() - possible_existing_entry_point_modules
)
# Generate new targets for entry points that don't already have one.
for entry_point_module in unowned_entry_point_modules:
entry_point = module_to_entry_point[entry_point_module]
path, fname = os.path.split(entry_point)
name = os.path.splitext(fname)[0]
pts.append(
PutativeTarget.for_target_type(
target_type=PexBinary,
path=path,
name=name,
triggering_sources=tuple(),
kwargs={"name": name, "entry_point": fname},
)
)
return PutativeTargets(pts)
