async def infer_python_dependencies( request: InferPythonDependencies, python_inference: PythonInference) -> InferredDependencies: if not python_inference.imports: return InferredDependencies([], sibling_dependencies_inferrable=False) stripped_sources = await Get(StrippedSourceFiles, SourceFilesRequest([request.sources_field])) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest) owner_requests: List[Get[PythonModuleOwner, PythonModule]] = [] for file_content, module in zip(digest_contents, modules): file_imports_obj = find_python_imports(file_content.content.decode(), module_name=module.module) detected_imports = (file_imports_obj.all_imports if python_inference.string_imports else file_imports_obj.explicit_imports) owner_requests.extend( Get(PythonModuleOwner, PythonModule(imported_module)) for imported_module in detected_imports if imported_module not in combined_stdlib) owner_per_import = await MultiGet(owner_requests) result = ( owner.address for owner in owner_per_import if owner.address and owner.address != request.sources_field.address) return InferredDependencies(result, sibling_dependencies_inferrable=True)
async def infer_python_dependencies( request: InferPythonDependencies, python_inference: PythonInference ) -> InferredDependencies: if not python_inference.imports: return InferredDependencies() stripped_sources = await Get(StrippedSourceFiles, SourceFilesRequest([request.sources_field])) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files ) digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest) imports_per_file = tuple( find_python_imports(file_content.content.decode(), module_name=module.module) for file_content, module in zip(digest_contents, modules) ) owner_per_import = await MultiGet( Get(PythonModuleOwner, PythonModule(imported_module)) for file_imports in imports_per_file for imported_module in file_imports.explicit_imports if imported_module not in combined_stdlib ) return InferredDependencies( owner.address for owner in owner_per_import if ( owner.address and owner.address.maybe_convert_to_base_target() != request.sources_field.address ) )
async def infer_python_dependencies( request: InferPythonDependencies) -> InferredDependencies: stripped_sources = await Get[SourceRootStrippedSources]( StripSourcesFieldRequest(request.sources_field)) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) files_content = await Get[FilesContent](Digest, stripped_sources.snapshot.digest) imports_per_file = tuple( find_python_imports(fc.content.decode(), module_name=module.module) for fc, module in zip(files_content, modules)) owner_per_import = await MultiGet( Get[PythonModuleOwner](PythonModule(imported_module)) for file_imports in imports_per_file for imported_module in file_imports.all_imports if imported_module not in combined_stdlib) return InferredDependencies( owner.address for owner in owner_per_import if owner.address and owner.address != request.sources_field.address)
async def infer_python_dependencies( request: InferPythonDependencies) -> InferredDependencies: stripped_sources = await Get[SourceRootStrippedSources]( StripSourcesFieldRequest(request.sources_field)) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) files_content = await Get[FilesContent](Digest, stripped_sources.snapshot.digest) imports_per_file = tuple( find_python_imports(fc.content.decode(), module_name=module.module) for fc, module in zip(files_content, modules)) owners_per_import = await MultiGet( Get[PythonModuleOwners](PythonModule(imported_module)) for file_imports in imports_per_file for imported_module in file_imports.all_imports if imported_module not in combined_stdlib) # We conservatively only use dep inference if there is exactly one owner for an import. return InferredDependencies( itertools.chain.from_iterable( owners for owners in owners_per_import if len(owners) == 1 and tuple(owners)[0] != request.sources_field.address))
async def infer_python_dependencies( request: InferPythonDependencies, python_inference: PythonInference) -> InferredDependencies: if not python_inference.imports: return InferredDependencies([], sibling_dependencies_inferrable=False) stripped_sources = await Get(StrippedSourceFiles, SourceFilesRequest([request.sources_field])) modules = tuple( PythonModule.create_from_stripped_path(PurePath(fp)) for fp in stripped_sources.snapshot.files) digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest) owners_requests: List[Get[PythonModuleOwners, PythonModule]] = [] for file_content, module in zip(digest_contents, modules): file_imports_obj = find_python_imports( filename=file_content.path, content=file_content.content.decode(), module_name=module.module, ) detected_imports = (file_imports_obj.all_imports if python_inference.string_imports else file_imports_obj.explicit_imports) owners_requests.extend( Get(PythonModuleOwners, PythonModule(imported_module)) for imported_module in detected_imports if imported_module not in combined_stdlib) owners_per_import = await MultiGet(owners_requests) # We remove the request's address so that we don't infer dependencies on self. merged_result = sorted( set(itertools.chain.from_iterable(owners_per_import)) - {request.sources_field.address}) return InferredDependencies(merged_result, sibling_dependencies_inferrable=True)
def test_create_module_from_path(stripped_path: PurePath, expected: str) -> None: assert PythonModule.create_from_stripped_path( stripped_path) == PythonModule(expected)
async def find_putative_targets( req: PutativePythonTargetsRequest, all_owned_sources: AllOwnedSources, python_setup: PythonSetup, ) -> PutativeTargets: # Find library/test/test_util targets. all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py") all_py_files = await Get(Paths, PathGlobs, all_py_files_globs) unowned_py_files = set(all_py_files.files) - set(all_owned_sources) classified_unowned_py_files = classify_source_files(unowned_py_files) pts = [] for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): if issubclass(tgt_type, PythonTestsGeneratorTarget): name = "tests" kwargs = {"name": name} elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget): name = "test_utils" kwargs = {"name": name} else: name = os.path.basename(dirname) kwargs = {} if ( python_setup.tailor_ignore_solitary_init_files and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"} ): continue pts.append( PutativeTarget.for_target_type( tgt_type, dirname, name, sorted(filenames), kwargs=kwargs ) ) if python_setup.tailor_requirements_targets: # Find requirements files. all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt") all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs) unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources) for req_file in unowned_requirements_files: path, name = os.path.split(req_file) pts.append( PutativeTarget( path=path, # python_requirements is a macro and doesn't take a name argument, but the # PutativeTarget still needs a name for display purposes. name=name, type_alias="python_requirements", triggering_sources=[req_file], owned_sources=[req_file], addressable=False, kwargs={} if name == "requirements.txt" else {"requirements_relpath": name}, ) ) if python_setup.tailor_pex_binary_targets: # Find binary targets. # Get all files whose content indicates that they are entry points. digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs) entry_points = [ file_content.path for file_content in digest_contents if is_entry_point(file_content.content) ] # Get the modules for these entry points. src_roots = await Get( SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points) ) module_to_entry_point = {} for entry_point in entry_points: entry_point_path = PurePath(entry_point) src_root = src_roots.path_to_root[entry_point_path] stripped_entry_point = entry_point_path.relative_to(src_root.path) module = PythonModule.create_from_stripped_path(stripped_entry_point) module_to_entry_point[module.module] = entry_point # Get existing binary targets for these entry points. entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points} possible_existing_binary_targets = await Get( UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs) ) possible_existing_binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField])) for t in possible_existing_binary_targets if t.has_field(PexEntryPointField) ) possible_existing_entry_point_modules = { rep.val.module for rep in possible_existing_binary_entry_points if rep.val } unowned_entry_point_modules = ( module_to_entry_point.keys() - possible_existing_entry_point_modules ) # Generate new targets for entry points that don't already have one. for entry_point_module in unowned_entry_point_modules: entry_point = module_to_entry_point[entry_point_module] path, fname = os.path.split(entry_point) name = os.path.splitext(fname)[0] pts.append( PutativeTarget.for_target_type( target_type=PexBinary, path=path, name=name, triggering_sources=tuple(), kwargs={"name": name, "entry_point": fname}, ) ) return PutativeTargets(pts)