def test_group_by_dir() -> None: paths = { "foo/bar/baz1.ext", "foo/bar/baz1_test.ext", "foo/bar/qux/quux1.ext", "foo/__init__.ext", "foo/bar/__init__.ext", "foo/bar/baz2.ext", "foo/bar1.ext", "foo1.ext", "__init__.ext", } assert { "": {"__init__.ext", "foo1.ext"}, "foo": {"__init__.ext", "bar1.ext"}, "foo/bar": {"__init__.ext", "baz1.ext", "baz1_test.ext", "baz2.ext"}, "foo/bar/qux": {"quux1.ext"}, } == group_by_dir(paths)
async def find_putative_go_module_targets( request: PutativeGoModuleTargetsRequest, all_owned_sources: AllOwnedSources ) -> PutativeTargets: all_go_mod_files = await Get(Paths, PathGlobs, request.search_paths.path_globs("go.mod")) unowned_go_mod_files = set(all_go_mod_files.files) - set(all_owned_sources) putative_targets = [] for dirname, filenames in group_by_dir(unowned_go_mod_files).items(): putative_targets.append( PutativeTarget.for_target_type( GoModule, dirname, os.path.basename(dirname), sorted(filenames), ) ) return PutativeTargets(putative_targets)
async def find_putative_targets( req: PutativeScalaTargetsRequest, all_owned_sources: AllOwnedSources, ) -> PutativeTargets: all_scala_files_globs = req.search_paths.path_globs("*.scala") all_scala_files = await Get(Paths, PathGlobs, all_scala_files_globs) unowned_scala_files = set(all_scala_files.files) - set(all_owned_sources) classified_unowned_scala_files = classify_source_files(unowned_scala_files) putative_targets = [] for tgt_type, paths in classified_unowned_scala_files.items(): for dirname, filenames in group_by_dir(paths).items(): putative_targets.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=None, triggering_sources=sorted(filenames) ) ) return PutativeTargets(putative_targets)
async def find_putative_thrift_targets( req: PutativeThriftTargetsRequest, all_owned_sources: AllOwnedSources, thrift_subsystem: ThriftSubsystem, ) -> PutativeTargets: if not thrift_subsystem.tailor: return PutativeTargets() all_thrift_files = await Get(Paths, PathGlobs, req.path_globs("*.thrift")) unowned_thrift_files = set(all_thrift_files.files) - set(all_owned_sources) pts = [ PutativeTarget.for_target_type( ThriftSourcesGeneratorTarget, path=dirname, name=None, triggering_sources=sorted(filenames), ) for dirname, filenames in group_by_dir(unowned_thrift_files).items() ] return PutativeTargets(pts)
async def find_putative_targets( _: PutativeShellTargetsRequest, all_owned_sources: AllOwnedSources) -> PutativeTargets: all_shell_files = await Get(Paths, PathGlobs(["**/*.sh"])) unowned_shell_files = set(all_shell_files.files) - set(all_owned_sources) classified_unowned_shell_files = classify_source_files(unowned_shell_files) pts = [] for tgt_type, paths in classified_unowned_shell_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == Shunit2Tests else os.path.basename( dirname) kwargs = {"name": name} if tgt_type == Shunit2Tests else {} pts.append( PutativeTarget.for_target_type(tgt_type, dirname, name, sorted(filenames), kwargs=kwargs)) return PutativeTargets(pts)
async def setup_scalafmt(request: ScalafmtRequest, ) -> Setup: toolcp_relpath = "__toolcp" lockfile_request = await Get(GenerateJvmLockfileFromTool, ScalafmtToolLockfileSentinel()) tool_classpath, config_files = await MultiGet( Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)), Get(ScalafmtConfigFiles, GatherScalafmtConfigFilesRequest(request.snapshot)), ) merged_sources_digest = await Get( Digest, MergeDigests([request.snapshot.digest, config_files.snapshot.digest])) extra_immutable_input_digests = { toolcp_relpath: tool_classpath.digest, } # Partition the work by which source files share the same config file (regardless of directory). source_files_by_config_file: dict[str, set[str]] = defaultdict(set) for source_dir, files_in_source_dir in group_by_dir( request.snapshot.files).items(): config_file = config_files.source_dir_to_config_file[source_dir] source_files_by_config_file[config_file].update( os.path.join(source_dir, name) for name in files_in_source_dir) partitions = await MultiGet( Get( Partition, SetupScalafmtPartition( classpath_entries=tuple( tool_classpath.classpath_entries(toolcp_relpath)), merged_sources_digest=merged_sources_digest, extra_immutable_input_digests=FrozenDict( extra_immutable_input_digests), config_file=config_file, files=tuple(sorted(files)), ), ) for config_file, files in source_files_by_config_file.items()) return Setup(tuple(partitions), original_snapshot=request.snapshot)
async def find_putative_targets(req: PutativeShellTargetsRequest, all_owned_sources: AllOwnedSources, shell_setup: ShellSetup) -> PutativeTargets: if not shell_setup.tailor: return PutativeTargets() all_shell_files = await Get(Paths, PathGlobs, req.path_globs("*.sh")) unowned_shell_files = set(all_shell_files.files) - set(all_owned_sources) classified_unowned_shell_files = classify_source_files(unowned_shell_files) pts = [] for tgt_type, paths in classified_unowned_shell_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == Shunit2TestsGeneratorTarget else None pts.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames))) return PutativeTargets(pts)
async def find_putative_targets( req: PutativeJSTargetsRequest, all_owned_sources: AllOwnedSources, ) -> PutativeTargets: all_js_files = await Get( Paths, PathGlobs, req.path_globs(*(f"*{ext}" for ext in JS_FILE_EXTENSIONS))) unowned_js_files = set(all_js_files.files) - set(all_owned_sources) classified_unowned_js_files = classify_source_files(unowned_js_files) putative_targets = [] for tgt_type, paths in classified_unowned_js_files.items(): for dirname, filenames in group_by_dir(paths).items(): putative_targets.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=None, triggering_sources=sorted(filenames))) return PutativeTargets(putative_targets)
async def find_putative_targets( req: PutativeJavaTargetsRequest, all_owned_sources: AllOwnedSources, ) -> PutativeTargets: all_java_files_globs = req.search_paths.path_globs("*.java") all_java_files = await Get(Paths, PathGlobs, all_java_files_globs) unowned_java_files = set(all_java_files.files) - set(all_owned_sources) classified_unowned_java_files = classify_source_files(unowned_java_files) putative_targets = [] for tgt_type, paths in classified_unowned_java_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == JunitTestsGeneratorTarget else os.path.basename(dirname) kwargs = {"name": name} if tgt_type == JunitTestsGeneratorTarget else {} putative_targets.append( PutativeTarget.for_target_type( tgt_type, dirname, name, sorted(filenames), kwargs=kwargs ) ) return PutativeTargets(putative_targets)
async def find_putative_targets( req: PutativeAvroTargetsRequest, all_owned_sources: AllOwnedSources) -> PutativeTargets: all_avsc_files, all_avpr_files, all_avdl_files = await MultiGet( Get(Paths, PathGlobs, req.search_paths.path_globs("*.avsc")), Get(Paths, PathGlobs, req.search_paths.path_globs("*.avpr")), Get(Paths, PathGlobs, req.search_paths.path_globs("*.avdl")), ) unowned_avro_files = { *all_avsc_files.files, *all_avpr_files.files, *all_avdl_files.files, } - set(all_owned_sources) pts = [ PutativeTarget.for_target_type( AvroSourcesGeneratorTarget, path=dirname, name=None, triggering_sources=sorted(filenames), ) for dirname, filenames in group_by_dir(unowned_avro_files).items() ] return PutativeTargets(pts)
async def find_putative_terrform_modules_targets( request: PutativeTerraformTargetsRequest, ) -> PutativeTargets: all_terraform_files = await Get(Paths, PathGlobs, request.search_paths.path_globs("*.tf")) directory_to_files = { dir: files for dir, files in group_by_dir(all_terraform_files.files).items() if any(file.endswith(".tf") for file in files) } prefixes = find_disjoint_longest_common_prefixes( [PurePath(dir).parts for dir in directory_to_files.keys()]) putative_targets = [ PutativeTarget.for_target_type( TerraformModulesGeneratorTarget, str(PurePath(*dir_parts)), "tf_mods", [str(PurePath(*dir_parts).joinpath("**/*.tf"))], ) for dir_parts in prefixes ] return PutativeTargets(putative_targets)
async def find_putative_targets(req: PutativeJavaTargetsRequest, all_owned_sources: AllOwnedSources, javac: JavacSubsystem) -> PutativeTargets: putative_targets = [] if javac.tailor_source_targets: all_java_files_globs = req.path_globs("*.java") all_java_files = await Get(Paths, PathGlobs, all_java_files_globs) unowned_java_files = set(all_java_files.files) - set(all_owned_sources) classified_unowned_java_files = classify_source_files( unowned_java_files) for tgt_type, paths in classified_unowned_java_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == JunitTestsGeneratorTarget else None putative_targets.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames))) return PutativeTargets(putative_targets)
async def find_fortran_targets( req: PutativeFortranTargetsRequest, all_owned_sources: AllOwnedSources) -> PutativeTargets: all_fortran_files = await Get(Paths, PathGlobs, req.search_paths.path_globs("*.f90")) unowned_shell_files = set(all_fortran_files.files) - set(all_owned_sources) tests_filespec = Filespec(includes=list(FortranTestsSources.default)) test_filenames = set( matches_filespec( tests_filespec, paths=[os.path.basename(path) for path in unowned_shell_files])) test_files = { path for path in unowned_shell_files if os.path.basename(path) in test_filenames } sources_files = set(unowned_shell_files) - test_files classified_unowned_shell_files = { FortranTestsTarget: test_files, FortranLibraryTarget: sources_files, } pts = [] for tgt_type, paths in classified_unowned_shell_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == FortranTestsTarget else os.path.basename( dirname) kwargs = {"name": name} if tgt_type == FortranTestsTarget else {} pts.append( PutativeTarget.for_target_type(tgt_type, dirname, name, sorted(filenames), kwargs=kwargs)) return PutativeTargets(pts)
async def find_putative_terrform_module_targets( request: PutativeTerraformTargetsRequest, terraform: TerraformTool, all_owned_sources: AllOwnedSources, ) -> PutativeTargets: if not terraform.tailor: return PutativeTargets() all_terraform_files = await Get(Paths, PathGlobs, request.path_globs("*.tf")) unowned_terraform_files = set( all_terraform_files.files) - set(all_owned_sources) putative_targets = [ PutativeTarget.for_target_type( TerraformModuleTarget, path=dirname, name=None, triggering_sources=sorted(filenames), ) for dirname, filenames in group_by_dir( unowned_terraform_files).items() ] return PutativeTargets(putative_targets)
async def find_putative_targets( req: PutativePythonTargetsRequest, all_owned_sources: AllOwnedSources, python_setup: PythonSetup, ) -> PutativeTargets: pts = [] if python_setup.tailor_source_targets: # Find library/test/test_util targets. all_py_files_globs: PathGlobs = req.path_globs("*.py", "*.pyi") all_py_files = await Get(Paths, PathGlobs, all_py_files_globs) unowned_py_files = set(all_py_files.files) - set(all_owned_sources) classified_unowned_py_files = classify_source_files(unowned_py_files) for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): name: str | None if issubclass(tgt_type, PythonTestsGeneratorTarget): name = "tests" elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget): name = "test_utils" else: name = None if (python_setup.tailor_ignore_solitary_init_files and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"}): continue pts.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames))) if python_setup.tailor_requirements_targets: # Find requirements files. ( all_requirements_files, all_pipenv_lockfile_files, all_pyproject_toml_contents, ) = await MultiGet( Get(DigestContents, PathGlobs, req.path_globs("*requirements*.txt")), Get(DigestContents, PathGlobs, req.path_globs("Pipfile.lock")), Get(DigestContents, PathGlobs, req.path_globs("pyproject.toml")), ) def add_req_targets(files: Iterable[FileContent], alias: str, target_name: str) -> None: contents = {i.path: i.content for i in files} unowned_files = set(contents) - set(all_owned_sources) for fp in unowned_files: path, name = os.path.split(fp) try: validate(fp, contents[fp], alias) except Exception as e: logger.warning( f"An error occurred when validating `{fp}`: {e}.\n\n" "You'll need to create targets for its contents manually.\n" "To silence this error in future, see " "https://www.pantsbuild.org/docs/reference-tailor#section-ignore-paths \n" ) continue pts.append( PutativeTarget( path=path, name=target_name, type_alias=alias, triggering_sources=[fp], owned_sources=[name], kwargs=({} if alias != "python_requirements" or name == "requirements.txt" else { "source": name }), )) def validate(path: str, contents: bytes, alias: str) -> None: if alias == "python_requirements": return validate_python_requirements(path, contents) elif alias == "pipenv_requirements": return validate_pipenv_requirements(contents) elif alias == "poetry_requirements": return validate_poetry_requirements(contents) def validate_python_requirements(path: str, contents: bytes) -> None: for _ in parse_requirements_file(contents.decode(), rel_path=path): pass def validate_pipenv_requirements(contents: bytes) -> None: parse_pipenv_requirements(contents) def validate_poetry_requirements(contents: bytes) -> None: p = PyProjectToml(PurePath(), PurePath(), contents.decode()) parse_pyproject_toml(p) add_req_targets(all_requirements_files, "python_requirements", "reqs") add_req_targets(all_pipenv_lockfile_files, "pipenv_requirements", "pipenv") add_req_targets( { fc for fc in all_pyproject_toml_contents if b"[tool.poetry" in fc.content }, "poetry_requirements", "poetry", ) if python_setup.tailor_pex_binary_targets: # Find binary targets. # Get all files whose content indicates that they are entry points or are __main__.py files. digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs) all_main_py = await Get(Paths, PathGlobs, req.path_globs("__main__.py")) entry_points = [ file_content.path for file_content in digest_contents if is_entry_point(file_content.content) ] + list(all_main_py.files) # Get the modules for these entry points. src_roots = await Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)) module_to_entry_point = {} for entry_point in entry_points: entry_point_path = PurePath(entry_point) src_root = src_roots.path_to_root[entry_point_path] stripped_entry_point = entry_point_path.relative_to(src_root.path) module = module_from_stripped_path(stripped_entry_point) module_to_entry_point[module] = entry_point # Get existing binary targets for these entry points. entry_point_dirs = { os.path.dirname(entry_point) for entry_point in entry_points } possible_existing_binary_targets = await Get( UnexpandedTargets, RawSpecs( ancestor_globs=tuple( AncestorGlobSpec(d) for d in entry_point_dirs), description_of_origin="the `pex_binary` tailor rule", ), ) possible_existing_binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField])) for t in possible_existing_binary_targets if t.has_field(PexEntryPointField)) possible_existing_entry_point_modules = { rep.val.module for rep in possible_existing_binary_entry_points if rep.val } unowned_entry_point_modules = (module_to_entry_point.keys() - possible_existing_entry_point_modules) # Generate new targets for entry points that don't already have one. for entry_point_module in unowned_entry_point_modules: entry_point = module_to_entry_point[entry_point_module] path, fname = os.path.split(entry_point) name = os.path.splitext(fname)[0] pts.append( PutativeTarget.for_target_type( target_type=PexBinary, path=path, name=name, triggering_sources=tuple(), kwargs={"entry_point": fname}, )) return PutativeTargets(pts)
async def setup_full_package_build_request( request: _SetupGoProtobufPackageBuildRequest, protoc: Protoc, go_protoc_plugin: _SetupGoProtocPlugin, package_mapping: ImportPathToPackages, go_protobuf_mapping: GoProtobufImportPathMapping, analyzer: PackageAnalyzerSetup, ) -> FallibleBuildGoPackageRequest: output_dir = "_generated_files" protoc_relpath = "__protoc" protoc_go_plugin_relpath = "__protoc_gen_go" transitive_targets, downloaded_protoc_binary, empty_output_dir = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)), Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)), Get(Digest, CreateDigest([Directory(output_dir)])), ) all_sources = await Get( SourceFiles, SourceFilesRequest( sources_fields=(tgt[ProtobufSourceField] for tgt in transitive_targets.closure), for_sources_types=(ProtobufSourceField, ), enable_codegen=True, ), ) source_roots, input_digest = await MultiGet( Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(all_sources.files)), Get(Digest, MergeDigests([all_sources.snapshot.digest, empty_output_dir])), ) source_root_paths = sorted( {sr.path for sr in source_roots.path_to_root.values()}) pkg_sources = await MultiGet( Get(SourcesPaths, SourcesPathsRequest(tgt[ProtobufSourceField])) for tgt in transitive_targets.roots) pkg_files = sorted({f for ps in pkg_sources for f in ps.files}) maybe_grpc_plugin_args = [] if any( tgt.get(ProtobufGrpcToggleField).value for tgt in transitive_targets.roots): maybe_grpc_plugin_args = [ f"--go-grpc_out={output_dir}", "--go-grpc_opt=paths=source_relative", ] gen_result = await Get( FallibleProcessResult, Process( argv=[ os.path.join(protoc_relpath, downloaded_protoc_binary.exe), f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}", f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}", f"--go_out={output_dir}", "--go_opt=paths=source_relative", *(f"--proto_path={source_root}" for source_root in source_root_paths), *maybe_grpc_plugin_args, *pkg_files, ], # Note: Necessary or else --plugin option needs absolute path. env={"PATH": protoc_go_plugin_relpath}, input_digest=input_digest, immutable_input_digests={ protoc_relpath: downloaded_protoc_binary.digest, protoc_go_plugin_relpath: go_protoc_plugin.digest, }, description=f"Generating Go sources from {request.import_path}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) if gen_result.exit_code != 0: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=gen_result.exit_code, stderr=gen_result.stderr.decode(), ) # Ensure that the generated files are in a single package directory. gen_sources = await Get(Snapshot, Digest, gen_result.output_digest) files_by_dir = group_by_dir(gen_sources.files) if len(files_by_dir) != 1: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=1, stderr= ("Expected Go files generated from Protobuf sources to be output to a single directory.\n" f"- import path: {request.import_path}\n" f"- protobuf files: {', '.join(pkg_files)}"), ) gen_dir = list(files_by_dir.keys())[0] # Analyze the generated sources. input_digest = await Get( Digest, MergeDigests([gen_sources.digest, analyzer.digest])) result = await Get( FallibleProcessResult, Process( (analyzer.path, gen_dir), input_digest=input_digest, description= f"Determine metadata for generated Go package for {request.import_path}", level=LogLevel.DEBUG, env={"CGO_ENABLED": "0"}, ), ) # Parse the metadata from the analysis. fallible_analysis = FallibleFirstPartyPkgAnalysis.from_process_result( result, dir_path=gen_dir, import_path=request.import_path, minimum_go_version="", description_of_source= f"Go package generated from protobuf targets `{', '.join(str(addr) for addr in request.addresses)}`", ) if not fallible_analysis.analysis: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=fallible_analysis.exit_code, stderr=fallible_analysis.stderr, ) analysis = fallible_analysis.analysis # Obtain build requests for third-party dependencies. # TODO: Consider how to merge this code with existing dependency inference code. dep_build_request_addrs: list[Address] = [] for dep_import_path in (*analysis.imports, *analysis.test_imports, *analysis.xtest_imports): # Infer dependencies on other Go packages. candidate_addresses = package_mapping.mapping.get(dep_import_path) if candidate_addresses: # TODO: Use explicit dependencies to disambiguate? This should never happen with Go backend though. if len(candidate_addresses) > 1: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=result.exit_code, stderr= (f"Multiple addresses match import of `{dep_import_path}`.\n" f"addresses: {', '.join(str(a) for a in candidate_addresses)}" ), ) dep_build_request_addrs.extend(candidate_addresses) # Infer dependencies on other generated Go sources. go_protobuf_candidate_addresses = go_protobuf_mapping.mapping.get( dep_import_path) if go_protobuf_candidate_addresses: dep_build_request_addrs.extend(go_protobuf_candidate_addresses) dep_build_requests = await MultiGet( Get(BuildGoPackageRequest, BuildGoPackageTargetRequest(addr)) for addr in dep_build_request_addrs) return FallibleBuildGoPackageRequest( request=BuildGoPackageRequest( import_path=request.import_path, digest=gen_sources.digest, dir_path=analysis.dir_path, go_file_names=analysis.go_files, s_file_names=analysis.s_files, direct_dependencies=dep_build_requests, minimum_go_version=analysis.minimum_go_version, ), import_path=request.import_path, )
async def analyze_go_third_party_module( request: AnalyzeThirdPartyModuleRequest, analyzer: PackageAnalyzerSetup, golang_subsystem: GolangSubsystem, ) -> AnalyzedThirdPartyModule: # Download the module. download_result = await Get( ProcessResult, GoSdkProcess( ("mod", "download", "-json", f"{request.name}@{request.version}"), input_digest=request.go_mod_digest, # for go.sum working_dir=os.path.dirname(request.go_mod_path) if request.go_mod_path else None, # Allow downloads of the module sources. allow_downloads=True, output_directories=("gopath", ), description=f"Download Go module {request.name}@{request.version}.", ), ) if len(download_result.stdout) == 0: raise AssertionError( f"Expected output from `go mod download` for {request.name}@{request.version}." ) module_metadata = json.loads(download_result.stdout) module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"], "gopath/") go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/") # Subset the output directory to just the module sources and go.mod (which may be generated). module_sources_snapshot = await Get( Snapshot, DigestSubset( download_result.output_digest, PathGlobs( [f"{module_sources_relpath}/**", go_mod_relpath], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= f"the download of Go module {request.name}@{request.version}", ), ), ) # Determine directories with potential Go packages in them. candidate_package_dirs = [] files_by_dir = group_by_dir(p for p in module_sources_snapshot.files if p.startswith(module_sources_relpath)) for maybe_pkg_dir, files in files_by_dir.items(): # Skip directories where "testdata" would end up in the import path. # See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585 if "testdata" in maybe_pkg_dir.split("/"): continue # Consider directories with at least one `.go` file as package candidates. if any(f for f in files if f.endswith(".go")): candidate_package_dirs.append(maybe_pkg_dir) candidate_package_dirs.sort() # Analyze all of the packages in this module. analyzer_relpath = "__analyzer" analysis_result = await Get( ProcessResult, Process( [ os.path.join(analyzer_relpath, analyzer.path), *candidate_package_dirs ], input_digest=module_sources_snapshot.digest, immutable_input_digests={ analyzer_relpath: analyzer.digest, }, description= f"Analyze metadata for Go third-party module: {request.name}@{request.version}", level=LogLevel.DEBUG, env={"CGO_ENABLED": "0"}, ), ) if len(analysis_result.stdout) == 0: return AnalyzedThirdPartyModule(FrozenOrderedSet()) package_analysis_gets = [] for pkg_path, pkg_json in zip( candidate_package_dirs, ijson.items(analysis_result.stdout, "", multiple_values=True)): package_analysis_gets.append( Get( FallibleThirdPartyPkgAnalysis, AnalyzeThirdPartyPackageRequest( pkg_json=_freeze_json_dict(pkg_json), module_sources_digest=module_sources_snapshot.digest, module_sources_path=module_sources_relpath, module_import_path=request.name, package_path=pkg_path, minimum_go_version=request.minimum_go_version, ), )) analyzed_packages_fallible = await MultiGet(package_analysis_gets) analyzed_packages = [ pkg.analysis for pkg in analyzed_packages_fallible if pkg.analysis and pkg.exit_code == 0 ] return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages))
async def generate_targets_from_go_mod( request: GenerateTargetsFromGoModRequest, files_not_found_behavior: FilesNotFoundBehavior, union_membership: UnionMembership, ) -> GeneratedTargets: generator_addr = request.generator.address go_mod_info, go_paths = await MultiGet( Get(GoModInfo, GoModInfoRequest(generator_addr)), Get( Paths, PathGlobs, request.generator[GoModPackageSourcesField].path_globs( files_not_found_behavior), ), ) all_third_party_packages = await Get( AllThirdPartyPackages, AllThirdPartyPackagesRequest(go_mod_info.stripped_digest), ) dir_to_filenames = group_by_dir(go_paths.files) matched_dirs = [ dir for dir, filenames in dir_to_filenames.items() if filenames ] def create_first_party_package_tgt(dir: str) -> GoFirstPartyPackageTarget: subpath = fast_relpath(dir, generator_addr.spec_path) import_path = f"{go_mod_info.import_path}/{subpath}" if subpath else go_mod_info.import_path return GoFirstPartyPackageTarget( { GoImportPathField.alias: import_path, GoFirstPartyPackageSubpathField.alias: subpath, GoFirstPartyPackageSourcesField.alias: tuple( sorted( os.path.join(subpath, f) for f in dir_to_filenames[dir])), }, # E.g. `src/go:mod#./subdir`. generator_addr.create_generated(f"./{subpath}"), union_membership, residence_dir=dir, ) first_party_pkgs = (create_first_party_package_tgt(dir) for dir in matched_dirs) def create_third_party_package_tgt( pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget: return GoThirdPartyPackageTarget( {GoImportPathField.alias: pkg_info.import_path}, # E.g. `src/go:mod#github.com/google/uuid`. generator_addr.create_generated(pkg_info.import_path), union_membership, residence_dir=generator_addr.spec_path, ) third_party_pkgs = (create_third_party_package_tgt(pkg_info) for pkg_info in all_third_party_packages. import_paths_to_pkg_info.values()) return GeneratedTargets(request.generator, (*first_party_pkgs, *third_party_pkgs))
async def find_putative_go_targets( request: PutativeGoTargetsRequest, all_owned_sources: AllOwnedSources ) -> PutativeTargets: putative_targets = [] all_go_mod_files, all_go_files, all_go_files_digest_contents = await MultiGet( Get(Paths, PathGlobs, request.search_paths.path_globs("go.mod")), Get(Paths, PathGlobs, request.search_paths.path_globs("*.go")), Get(DigestContents, PathGlobs, request.search_paths.path_globs("*.go")), ) # Add `go_mod` targets. unowned_go_mod_files = set(all_go_mod_files.files) - set(all_owned_sources) for dirname, filenames in group_by_dir(unowned_go_mod_files).items(): putative_targets.append( PutativeTarget.for_target_type( GoModTarget, path=dirname, name=None, triggering_sources=sorted(filenames), ) ) # Add `go_package` targets. unowned_go_files = set(all_go_files.files) - set(all_owned_sources) for dirname, filenames in group_by_dir(unowned_go_files).items(): # Ignore paths that have `testdata` or `vendor` in them. # From `go help packages`: Note, however, that a directory named vendor that itself contains code # is not a vendored package: cmd/vendor would be a command named vendor. dirname_parts = PurePath(dirname).parts if "testdata" in dirname_parts or "vendor" in dirname_parts[0:-1]: continue putative_targets.append( PutativeTarget.for_target_type( GoPackageTarget, path=dirname, name=None, triggering_sources=sorted(filenames), ) ) # Add `go_binary` targets. main_package_dirs = [ os.path.dirname(file_content.path) for file_content in all_go_files_digest_contents if has_package_main(file_content.content) ] existing_targets = await Get( UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in main_package_dirs) ) owned_main_packages = await MultiGet( Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(t[GoBinaryMainPackageField])) for t in existing_targets if t.has_field(GoBinaryMainPackageField) ) unowned_main_package_dirs = set(main_package_dirs) - { # NB: We assume the `go_package` lives in the directory it's defined, which we validate # by e.g. banning `**` in its sources field. pkg.address.spec_path for pkg in owned_main_packages } putative_targets.extend( PutativeTarget.for_target_type( GoBinaryTarget, path=main_pkg_dir, name="bin", triggering_sources=tuple(), ) for main_pkg_dir in unowned_main_package_dirs ) return PutativeTargets(putative_targets)
async def find_putative_go_targets( request: PutativeGoTargetsRequest, all_owned_sources: AllOwnedSources, golang_subsystem: GolangSubsystem, ) -> PutativeTargets: putative_targets = [] _all_go_mod_paths = await Get(Paths, PathGlobs, request.path_globs("go.mod")) all_go_mod_files = set(_all_go_mod_paths.files) all_go_mod_dirs = {os.path.dirname(fp) for fp in all_go_mod_files} if golang_subsystem.tailor_go_mod_targets: unowned_go_mod_files = all_go_mod_files - set(all_owned_sources) for dirname, filenames in group_by_dir(unowned_go_mod_files).items(): putative_targets.append( PutativeTarget.for_target_type( GoModTarget, path=dirname, name=None, triggering_sources=sorted(filenames), )) if golang_subsystem.tailor_package_targets: all_go_files = await Get(Paths, PathGlobs, request.path_globs("*.go")) unowned_go_files = set(all_go_files.files) - set(all_owned_sources) for dirname, filenames in group_by_dir(unowned_go_files).items(): # Ignore paths that have `testdata` or `vendor` in them. # From `go help packages`: Note, however, that a directory named vendor that itself # contains code is not a vendored package: cmd/vendor would be a command named vendor. dirname_parts = PurePath(dirname).parts if "testdata" in dirname_parts or "vendor" in dirname_parts[0:-1]: continue if not has_go_mod_ancestor(dirname, all_go_mod_dirs): continue putative_targets.append( PutativeTarget.for_target_type( GoPackageTarget, path=dirname, name=None, triggering_sources=sorted(filenames), )) if golang_subsystem.tailor_binary_targets: all_go_files_digest_contents = await Get(DigestContents, PathGlobs, request.path_globs("*.go")) main_package_dirs = [] for file_content in all_go_files_digest_contents: dirname = os.path.dirname(file_content.path) if has_package_main(file_content.content) and has_go_mod_ancestor( dirname, all_go_mod_dirs): main_package_dirs.append(dirname) existing_targets = await Get( UnexpandedTargets, RawSpecs( ancestor_globs=tuple( AncestorGlobSpec(d) for d in main_package_dirs), description_of_origin="the `go_binary` tailor rule", ), ) owned_main_packages = await MultiGet( Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(t[GoBinaryMainPackageField])) for t in existing_targets if t.has_field(GoBinaryMainPackageField)) unowned_main_package_dirs = set(main_package_dirs) - { # NB: We assume the `go_package` lives in the directory it's defined, which we validate # by e.g. banning `**` in its sources field. pkg.address.spec_path for pkg in owned_main_packages } putative_targets.extend( PutativeTarget.for_target_type( GoBinaryTarget, path=main_pkg_dir, name="bin", triggering_sources=tuple(), ) for main_pkg_dir in unowned_main_package_dirs) return PutativeTargets(putative_targets)
async def find_putative_targets( req: PutativePythonTargetsRequest, all_owned_sources: AllOwnedSources, python_setup: PythonSetup, ) -> PutativeTargets: # Find library/test/test_util targets. all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py") all_py_files = await Get(Paths, PathGlobs, all_py_files_globs) unowned_py_files = set(all_py_files.files) - set(all_owned_sources) classified_unowned_py_files = classify_source_files(unowned_py_files) pts = [] for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): if issubclass(tgt_type, PythonTestsGeneratorTarget): name = "tests" kwargs = {"name": name} elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget): name = "test_utils" kwargs = {"name": name} else: name = os.path.basename(dirname) kwargs = {} if ( python_setup.tailor_ignore_solitary_init_files and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"} ): continue pts.append( PutativeTarget.for_target_type( tgt_type, dirname, name, sorted(filenames), kwargs=kwargs ) ) if python_setup.tailor_requirements_targets: # Find requirements files. all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt") all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs) unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources) for req_file in unowned_requirements_files: path, name = os.path.split(req_file) pts.append( PutativeTarget( path=path, # python_requirements is a macro and doesn't take a name argument, but the # PutativeTarget still needs a name for display purposes. name=name, type_alias="python_requirements", triggering_sources=[req_file], owned_sources=[req_file], addressable=False, kwargs={} if name == "requirements.txt" else {"requirements_relpath": name}, ) ) if python_setup.tailor_pex_binary_targets: # Find binary targets. # Get all files whose content indicates that they are entry points. digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs) entry_points = [ file_content.path for file_content in digest_contents if is_entry_point(file_content.content) ] # Get the modules for these entry points. src_roots = await Get( SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points) ) module_to_entry_point = {} for entry_point in entry_points: entry_point_path = PurePath(entry_point) src_root = src_roots.path_to_root[entry_point_path] stripped_entry_point = entry_point_path.relative_to(src_root.path) module = PythonModule.create_from_stripped_path(stripped_entry_point) module_to_entry_point[module.module] = entry_point # Get existing binary targets for these entry points. entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points} possible_existing_binary_targets = await Get( UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs) ) possible_existing_binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField])) for t in possible_existing_binary_targets if t.has_field(PexEntryPointField) ) possible_existing_entry_point_modules = { rep.val.module for rep in possible_existing_binary_entry_points if rep.val } unowned_entry_point_modules = ( module_to_entry_point.keys() - possible_existing_entry_point_modules ) # Generate new targets for entry points that don't already have one. for entry_point_module in unowned_entry_point_modules: entry_point = module_to_entry_point[entry_point_module] path, fname = os.path.split(entry_point) name = os.path.splitext(fname)[0] pts.append( PutativeTarget.for_target_type( target_type=PexBinary, path=path, name=name, triggering_sources=tuple(), kwargs={"name": name, "entry_point": fname}, ) ) return PutativeTargets(pts)
async def collect_fixture_configs( _request: CollectFixtureConfigsRequest, pytest: PyTest, python_setup: PythonSetup, test_extra_env: TestExtraEnv, targets: Targets, ) -> CollectedJVMLockfileFixtureConfigs: addresses = [tgt.address for tgt in targets] transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) all_targets = transitive_targets.closure interpreter_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup) pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet( Get( Pex, PexRequest( output_filename="pytest.pex", requirements=pytest.pex_requirements(), interpreter_constraints=interpreter_constraints, internal_only=True, ), ), Get(Pex, RequirementsPexRequest(addresses)), Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True, include_resources=True), ), Get( PythonSourceFiles, PythonSourceFilesRequest(targets), ), ) script_content = FileContent(path="collect-fixtures.py", content=COLLECTION_SCRIPT.encode(), is_executable=True) script_digest = await Get(Digest, CreateDigest([script_content])) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=EntryPoint(PurePath(script_content.path).stem), sources=script_digest, internal_only=True, pex_path=[ pytest_pex, requirements_pex, ], ), ) config_file_dirs = list( group_by_dir(prepared_sources.source_files.files).keys()) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(config_file_dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) pytest_config_digest = config_files.snapshot.digest input_digest = await Get( Digest, MergeDigests(( prepared_sources.source_files.snapshot.digest, pytest_config_digest, )), ) extra_env = { "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=[ name for name in root_sources.source_files.files if name.endswith(".py") ], extra_env=extra_env, input_digest=input_digest, output_files=("tests.json", ), description="Collect test lockfile requirements from all tests.", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_SESSION, ), ) result = await Get(ProcessResult, Process, process) digest_contents = await Get(DigestContents, Digest, result.output_digest) assert len(digest_contents) == 1 assert digest_contents[0].path == "tests.json" raw_config_data = json.loads(digest_contents[0].content) configs = [] for item in raw_config_data: config = JVMLockfileFixtureConfig( definition=JVMLockfileFixtureDefinition.from_kwargs( item["kwargs"]), test_file_path=item["test_file_path"], ) configs.append(config) return CollectedJVMLockfileFixtureConfigs(configs)