def test_group_by_dir() -> None:
paths = {
"foo/bar/baz1.ext",
"foo/bar/baz1_test.ext",
"foo/bar/qux/quux1.ext",
"foo/__init__.ext",
"foo/bar/__init__.ext",
"foo/bar/baz2.ext",
"foo/bar1.ext",
"foo1.ext",
"__init__.ext",
}
assert {
"": {"__init__.ext", "foo1.ext"},
"foo": {"__init__.ext", "bar1.ext"},
"foo/bar": {"__init__.ext", "baz1.ext", "baz1_test.ext", "baz2.ext"},
"foo/bar/qux": {"quux1.ext"},
} == group_by_dir(paths)
async def find_putative_go_module_targets(
request: PutativeGoModuleTargetsRequest, all_owned_sources: AllOwnedSources
) -> PutativeTargets:
all_go_mod_files = await Get(Paths, PathGlobs, request.search_paths.path_globs("go.mod"))
unowned_go_mod_files = set(all_go_mod_files.files) - set(all_owned_sources)
putative_targets = []
for dirname, filenames in group_by_dir(unowned_go_mod_files).items():
putative_targets.append(
PutativeTarget.for_target_type(
GoModule,
dirname,
os.path.basename(dirname),
sorted(filenames),
)
)
return PutativeTargets(putative_targets)
async def find_putative_targets(
req: PutativeScalaTargetsRequest,
all_owned_sources: AllOwnedSources,
) -> PutativeTargets:
all_scala_files_globs = req.search_paths.path_globs("*.scala")
all_scala_files = await Get(Paths, PathGlobs, all_scala_files_globs)
unowned_scala_files = set(all_scala_files.files) - set(all_owned_sources)
classified_unowned_scala_files = classify_source_files(unowned_scala_files)
putative_targets = []
for tgt_type, paths in classified_unowned_scala_files.items():
for dirname, filenames in group_by_dir(paths).items():
putative_targets.append(
PutativeTarget.for_target_type(
tgt_type, path=dirname, name=None, triggering_sources=sorted(filenames)
)
)
return PutativeTargets(putative_targets)
async def find_putative_thrift_targets(
req: PutativeThriftTargetsRequest,
all_owned_sources: AllOwnedSources,
thrift_subsystem: ThriftSubsystem,
) -> PutativeTargets:
if not thrift_subsystem.tailor:
return PutativeTargets()
all_thrift_files = await Get(Paths, PathGlobs, req.path_globs("*.thrift"))
unowned_thrift_files = set(all_thrift_files.files) - set(all_owned_sources)
pts = [
PutativeTarget.for_target_type(
ThriftSourcesGeneratorTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
) for dirname, filenames in group_by_dir(unowned_thrift_files).items()
]
return PutativeTargets(pts)
async def find_putative_targets(
_: PutativeShellTargetsRequest,
all_owned_sources: AllOwnedSources) -> PutativeTargets:
all_shell_files = await Get(Paths, PathGlobs(["**/*.sh"]))
unowned_shell_files = set(all_shell_files.files) - set(all_owned_sources)
classified_unowned_shell_files = classify_source_files(unowned_shell_files)
pts = []
for tgt_type, paths in classified_unowned_shell_files.items():
for dirname, filenames in group_by_dir(paths).items():
name = "tests" if tgt_type == Shunit2Tests else os.path.basename(
dirname)
kwargs = {"name": name} if tgt_type == Shunit2Tests else {}
pts.append(
PutativeTarget.for_target_type(tgt_type,
dirname,
name,
sorted(filenames),
kwargs=kwargs))
return PutativeTargets(pts)
async def setup_scalafmt(request: ScalafmtRequest, ) -> Setup:
toolcp_relpath = "__toolcp"
lockfile_request = await Get(GenerateJvmLockfileFromTool,
ScalafmtToolLockfileSentinel())
tool_classpath, config_files = await MultiGet(
Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
Get(ScalafmtConfigFiles,
GatherScalafmtConfigFilesRequest(request.snapshot)),
)
merged_sources_digest = await Get(
Digest,
MergeDigests([request.snapshot.digest, config_files.snapshot.digest]))
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
}
# Partition the work by which source files share the same config file (regardless of directory).
source_files_by_config_file: dict[str, set[str]] = defaultdict(set)
for source_dir, files_in_source_dir in group_by_dir(
request.snapshot.files).items():
config_file = config_files.source_dir_to_config_file[source_dir]
source_files_by_config_file[config_file].update(
os.path.join(source_dir, name) for name in files_in_source_dir)
partitions = await MultiGet(
Get(
Partition,
SetupScalafmtPartition(
classpath_entries=tuple(
tool_classpath.classpath_entries(toolcp_relpath)),
merged_sources_digest=merged_sources_digest,
extra_immutable_input_digests=FrozenDict(
extra_immutable_input_digests),
config_file=config_file,
files=tuple(sorted(files)),
),
) for config_file, files in source_files_by_config_file.items())
return Setup(tuple(partitions), original_snapshot=request.snapshot)
async def find_putative_targets(req: PutativeShellTargetsRequest,
all_owned_sources: AllOwnedSources,
shell_setup: ShellSetup) -> PutativeTargets:
if not shell_setup.tailor:
return PutativeTargets()
all_shell_files = await Get(Paths, PathGlobs, req.path_globs("*.sh"))
unowned_shell_files = set(all_shell_files.files) - set(all_owned_sources)
classified_unowned_shell_files = classify_source_files(unowned_shell_files)
pts = []
for tgt_type, paths in classified_unowned_shell_files.items():
for dirname, filenames in group_by_dir(paths).items():
name = "tests" if tgt_type == Shunit2TestsGeneratorTarget else None
pts.append(
PutativeTarget.for_target_type(
tgt_type,
path=dirname,
name=name,
triggering_sources=sorted(filenames)))
return PutativeTargets(pts)
async def find_putative_targets(
req: PutativeJSTargetsRequest,
all_owned_sources: AllOwnedSources,
) -> PutativeTargets:
all_js_files = await Get(
Paths, PathGlobs,
req.path_globs(*(f"*{ext}" for ext in JS_FILE_EXTENSIONS)))
unowned_js_files = set(all_js_files.files) - set(all_owned_sources)
classified_unowned_js_files = classify_source_files(unowned_js_files)
putative_targets = []
for tgt_type, paths in classified_unowned_js_files.items():
for dirname, filenames in group_by_dir(paths).items():
putative_targets.append(
PutativeTarget.for_target_type(
tgt_type,
path=dirname,
name=None,
triggering_sources=sorted(filenames)))
return PutativeTargets(putative_targets)
async def find_putative_targets(
req: PutativeJavaTargetsRequest,
all_owned_sources: AllOwnedSources,
) -> PutativeTargets:
all_java_files_globs = req.search_paths.path_globs("*.java")
all_java_files = await Get(Paths, PathGlobs, all_java_files_globs)
unowned_java_files = set(all_java_files.files) - set(all_owned_sources)
classified_unowned_java_files = classify_source_files(unowned_java_files)
putative_targets = []
for tgt_type, paths in classified_unowned_java_files.items():
for dirname, filenames in group_by_dir(paths).items():
name = "tests" if tgt_type == JunitTestsGeneratorTarget else os.path.basename(dirname)
kwargs = {"name": name} if tgt_type == JunitTestsGeneratorTarget else {}
putative_targets.append(
PutativeTarget.for_target_type(
tgt_type, dirname, name, sorted(filenames), kwargs=kwargs
)
)
return PutativeTargets(putative_targets)
async def find_putative_targets(
req: PutativeAvroTargetsRequest,
all_owned_sources: AllOwnedSources) -> PutativeTargets:
all_avsc_files, all_avpr_files, all_avdl_files = await MultiGet(
Get(Paths, PathGlobs, req.search_paths.path_globs("*.avsc")),
Get(Paths, PathGlobs, req.search_paths.path_globs("*.avpr")),
Get(Paths, PathGlobs, req.search_paths.path_globs("*.avdl")),
)
unowned_avro_files = {
*all_avsc_files.files,
*all_avpr_files.files,
*all_avdl_files.files,
} - set(all_owned_sources)
pts = [
PutativeTarget.for_target_type(
AvroSourcesGeneratorTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
) for dirname, filenames in group_by_dir(unowned_avro_files).items()
]
return PutativeTargets(pts)
async def find_putative_terrform_modules_targets(
request: PutativeTerraformTargetsRequest, ) -> PutativeTargets:
all_terraform_files = await Get(Paths, PathGlobs,
request.search_paths.path_globs("*.tf"))
directory_to_files = {
dir: files
for dir, files in group_by_dir(all_terraform_files.files).items()
if any(file.endswith(".tf") for file in files)
}
prefixes = find_disjoint_longest_common_prefixes(
[PurePath(dir).parts for dir in directory_to_files.keys()])
putative_targets = [
PutativeTarget.for_target_type(
TerraformModulesGeneratorTarget,
str(PurePath(*dir_parts)),
"tf_mods",
[str(PurePath(*dir_parts).joinpath("**/*.tf"))],
) for dir_parts in prefixes
]
return PutativeTargets(putative_targets)
async def find_putative_targets(req: PutativeJavaTargetsRequest,
all_owned_sources: AllOwnedSources,
javac: JavacSubsystem) -> PutativeTargets:
putative_targets = []
if javac.tailor_source_targets:
all_java_files_globs = req.path_globs("*.java")
all_java_files = await Get(Paths, PathGlobs, all_java_files_globs)
unowned_java_files = set(all_java_files.files) - set(all_owned_sources)
classified_unowned_java_files = classify_source_files(
unowned_java_files)
for tgt_type, paths in classified_unowned_java_files.items():
for dirname, filenames in group_by_dir(paths).items():
name = "tests" if tgt_type == JunitTestsGeneratorTarget else None
putative_targets.append(
PutativeTarget.for_target_type(
tgt_type,
path=dirname,
name=name,
triggering_sources=sorted(filenames)))
return PutativeTargets(putative_targets)
async def find_fortran_targets(
req: PutativeFortranTargetsRequest,
all_owned_sources: AllOwnedSources) -> PutativeTargets:
all_fortran_files = await Get(Paths, PathGlobs,
req.search_paths.path_globs("*.f90"))
unowned_shell_files = set(all_fortran_files.files) - set(all_owned_sources)
tests_filespec = Filespec(includes=list(FortranTestsSources.default))
test_filenames = set(
matches_filespec(
tests_filespec,
paths=[os.path.basename(path) for path in unowned_shell_files]))
test_files = {
path
for path in unowned_shell_files
if os.path.basename(path) in test_filenames
}
sources_files = set(unowned_shell_files) - test_files
classified_unowned_shell_files = {
FortranTestsTarget: test_files,
FortranLibraryTarget: sources_files,
}
pts = []
for tgt_type, paths in classified_unowned_shell_files.items():
for dirname, filenames in group_by_dir(paths).items():
name = "tests" if tgt_type == FortranTestsTarget else os.path.basename(
dirname)
kwargs = {"name": name} if tgt_type == FortranTestsTarget else {}
pts.append(
PutativeTarget.for_target_type(tgt_type,
dirname,
name,
sorted(filenames),
kwargs=kwargs))
return PutativeTargets(pts)
async def find_putative_terrform_module_targets(
request: PutativeTerraformTargetsRequest,
terraform: TerraformTool,
all_owned_sources: AllOwnedSources,
) -> PutativeTargets:
if not terraform.tailor:
return PutativeTargets()
all_terraform_files = await Get(Paths, PathGlobs,
request.path_globs("*.tf"))
unowned_terraform_files = set(
all_terraform_files.files) - set(all_owned_sources)
putative_targets = [
PutativeTarget.for_target_type(
TerraformModuleTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
) for dirname, filenames in group_by_dir(
unowned_terraform_files).items()
]
return PutativeTargets(putative_targets)
async def find_putative_targets(
req: PutativePythonTargetsRequest,
all_owned_sources: AllOwnedSources,
python_setup: PythonSetup,
) -> PutativeTargets:
pts = []
if python_setup.tailor_source_targets:
# Find library/test/test_util targets.
all_py_files_globs: PathGlobs = req.path_globs("*.py", "*.pyi")
all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
classified_unowned_py_files = classify_source_files(unowned_py_files)
for tgt_type, paths in classified_unowned_py_files.items():
for dirname, filenames in group_by_dir(paths).items():
name: str | None
if issubclass(tgt_type, PythonTestsGeneratorTarget):
name = "tests"
elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
name = "test_utils"
else:
name = None
if (python_setup.tailor_ignore_solitary_init_files
and tgt_type == PythonSourcesGeneratorTarget
and filenames == {"__init__.py"}):
continue
pts.append(
PutativeTarget.for_target_type(
tgt_type,
path=dirname,
name=name,
triggering_sources=sorted(filenames)))
if python_setup.tailor_requirements_targets:
# Find requirements files.
(
all_requirements_files,
all_pipenv_lockfile_files,
all_pyproject_toml_contents,
) = await MultiGet(
Get(DigestContents, PathGlobs,
req.path_globs("*requirements*.txt")),
Get(DigestContents, PathGlobs, req.path_globs("Pipfile.lock")),
Get(DigestContents, PathGlobs, req.path_globs("pyproject.toml")),
)
def add_req_targets(files: Iterable[FileContent], alias: str,
target_name: str) -> None:
contents = {i.path: i.content for i in files}
unowned_files = set(contents) - set(all_owned_sources)
for fp in unowned_files:
path, name = os.path.split(fp)
try:
validate(fp, contents[fp], alias)
except Exception as e:
logger.warning(
f"An error occurred when validating `{fp}`: {e}.\n\n"
"You'll need to create targets for its contents manually.\n"
"To silence this error in future, see "
"https://www.pantsbuild.org/docs/reference-tailor#section-ignore-paths \n"
)
continue
pts.append(
PutativeTarget(
path=path,
name=target_name,
type_alias=alias,
triggering_sources=[fp],
owned_sources=[name],
kwargs=({} if alias != "python_requirements"
or name == "requirements.txt" else {
"source": name
}),
))
def validate(path: str, contents: bytes, alias: str) -> None:
if alias == "python_requirements":
return validate_python_requirements(path, contents)
elif alias == "pipenv_requirements":
return validate_pipenv_requirements(contents)
elif alias == "poetry_requirements":
return validate_poetry_requirements(contents)
def validate_python_requirements(path: str, contents: bytes) -> None:
for _ in parse_requirements_file(contents.decode(), rel_path=path):
pass
def validate_pipenv_requirements(contents: bytes) -> None:
parse_pipenv_requirements(contents)
def validate_poetry_requirements(contents: bytes) -> None:
p = PyProjectToml(PurePath(), PurePath(), contents.decode())
parse_pyproject_toml(p)
add_req_targets(all_requirements_files, "python_requirements", "reqs")
add_req_targets(all_pipenv_lockfile_files, "pipenv_requirements",
"pipenv")
add_req_targets(
{
fc
for fc in all_pyproject_toml_contents
if b"[tool.poetry" in fc.content
},
"poetry_requirements",
"poetry",
)
if python_setup.tailor_pex_binary_targets:
# Find binary targets.
# Get all files whose content indicates that they are entry points or are __main__.py files.
digest_contents = await Get(DigestContents, PathGlobs,
all_py_files_globs)
all_main_py = await Get(Paths, PathGlobs,
req.path_globs("__main__.py"))
entry_points = [
file_content.path for file_content in digest_contents
if is_entry_point(file_content.content)
] + list(all_main_py.files)
# Get the modules for these entry points.
src_roots = await Get(SourceRootsResult, SourceRootsRequest,
SourceRootsRequest.for_files(entry_points))
module_to_entry_point = {}
for entry_point in entry_points:
entry_point_path = PurePath(entry_point)
src_root = src_roots.path_to_root[entry_point_path]
stripped_entry_point = entry_point_path.relative_to(src_root.path)
module = module_from_stripped_path(stripped_entry_point)
module_to_entry_point[module] = entry_point
# Get existing binary targets for these entry points.
entry_point_dirs = {
os.path.dirname(entry_point)
for entry_point in entry_points
}
possible_existing_binary_targets = await Get(
UnexpandedTargets,
RawSpecs(
ancestor_globs=tuple(
AncestorGlobSpec(d) for d in entry_point_dirs),
description_of_origin="the `pex_binary` tailor rule",
),
)
possible_existing_binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(t[PexEntryPointField]))
for t in possible_existing_binary_targets
if t.has_field(PexEntryPointField))
possible_existing_entry_point_modules = {
rep.val.module
for rep in possible_existing_binary_entry_points if rep.val
}
unowned_entry_point_modules = (module_to_entry_point.keys() -
possible_existing_entry_point_modules)
# Generate new targets for entry points that don't already have one.
for entry_point_module in unowned_entry_point_modules:
entry_point = module_to_entry_point[entry_point_module]
path, fname = os.path.split(entry_point)
name = os.path.splitext(fname)[0]
pts.append(
PutativeTarget.for_target_type(
target_type=PexBinary,
path=path,
name=name,
triggering_sources=tuple(),
kwargs={"entry_point": fname},
))
return PutativeTargets(pts)
async def setup_full_package_build_request(
request: _SetupGoProtobufPackageBuildRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
package_mapping: ImportPathToPackages,
go_protobuf_mapping: GoProtobufImportPathMapping,
analyzer: PackageAnalyzerSetup,
) -> FallibleBuildGoPackageRequest:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
transitive_targets, downloaded_protoc_binary, empty_output_dir = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
)
all_sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSourceField, ),
enable_codegen=True,
),
)
source_roots, input_digest = await MultiGet(
Get(SourceRootsResult, SourceRootsRequest,
SourceRootsRequest.for_files(all_sources.files)),
Get(Digest,
MergeDigests([all_sources.snapshot.digest, empty_output_dir])),
)
source_root_paths = sorted(
{sr.path
for sr in source_roots.path_to_root.values()})
pkg_sources = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(tgt[ProtobufSourceField]))
for tgt in transitive_targets.roots)
pkg_files = sorted({f for ps in pkg_sources for f in ps.files})
maybe_grpc_plugin_args = []
if any(
tgt.get(ProtobufGrpcToggleField).value
for tgt in transitive_targets.roots):
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
gen_result = await Get(
FallibleProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*(f"--proto_path={source_root}"
for source_root in source_root_paths),
*maybe_grpc_plugin_args,
*pkg_files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=f"Generating Go sources from {request.import_path}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
if gen_result.exit_code != 0:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=gen_result.exit_code,
stderr=gen_result.stderr.decode(),
)
# Ensure that the generated files are in a single package directory.
gen_sources = await Get(Snapshot, Digest, gen_result.output_digest)
files_by_dir = group_by_dir(gen_sources.files)
if len(files_by_dir) != 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=1,
stderr=
("Expected Go files generated from Protobuf sources to be output to a single directory.\n"
f"- import path: {request.import_path}\n"
f"- protobuf files: {', '.join(pkg_files)}"),
)
gen_dir = list(files_by_dir.keys())[0]
# Analyze the generated sources.
input_digest = await Get(
Digest, MergeDigests([gen_sources.digest, analyzer.digest]))
result = await Get(
FallibleProcessResult,
Process(
(analyzer.path, gen_dir),
input_digest=input_digest,
description=
f"Determine metadata for generated Go package for {request.import_path}",
level=LogLevel.DEBUG,
env={"CGO_ENABLED": "0"},
),
)
# Parse the metadata from the analysis.
fallible_analysis = FallibleFirstPartyPkgAnalysis.from_process_result(
result,
dir_path=gen_dir,
import_path=request.import_path,
minimum_go_version="",
description_of_source=
f"Go package generated from protobuf targets `{', '.join(str(addr) for addr in request.addresses)}`",
)
if not fallible_analysis.analysis:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=fallible_analysis.exit_code,
stderr=fallible_analysis.stderr,
)
analysis = fallible_analysis.analysis
# Obtain build requests for third-party dependencies.
# TODO: Consider how to merge this code with existing dependency inference code.
dep_build_request_addrs: list[Address] = []
for dep_import_path in (*analysis.imports, *analysis.test_imports,
*analysis.xtest_imports):
# Infer dependencies on other Go packages.
candidate_addresses = package_mapping.mapping.get(dep_import_path)
if candidate_addresses:
# TODO: Use explicit dependencies to disambiguate? This should never happen with Go backend though.
if len(candidate_addresses) > 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=result.exit_code,
stderr=
(f"Multiple addresses match import of `{dep_import_path}`.\n"
f"addresses: {', '.join(str(a) for a in candidate_addresses)}"
),
)
dep_build_request_addrs.extend(candidate_addresses)
# Infer dependencies on other generated Go sources.
go_protobuf_candidate_addresses = go_protobuf_mapping.mapping.get(
dep_import_path)
if go_protobuf_candidate_addresses:
dep_build_request_addrs.extend(go_protobuf_candidate_addresses)
dep_build_requests = await MultiGet(
Get(BuildGoPackageRequest, BuildGoPackageTargetRequest(addr))
for addr in dep_build_request_addrs)
return FallibleBuildGoPackageRequest(
request=BuildGoPackageRequest(
import_path=request.import_path,
digest=gen_sources.digest,
dir_path=analysis.dir_path,
go_file_names=analysis.go_files,
s_file_names=analysis.s_files,
direct_dependencies=dep_build_requests,
minimum_go_version=analysis.minimum_go_version,
),
import_path=request.import_path,
)
async def analyze_go_third_party_module(
request: AnalyzeThirdPartyModuleRequest,
analyzer: PackageAnalyzerSetup,
golang_subsystem: GolangSubsystem,
) -> AnalyzedThirdPartyModule:
# Download the module.
download_result = await Get(
ProcessResult,
GoSdkProcess(
("mod", "download", "-json", f"{request.name}@{request.version}"),
input_digest=request.go_mod_digest, # for go.sum
working_dir=os.path.dirname(request.go_mod_path)
if request.go_mod_path else None,
# Allow downloads of the module sources.
allow_downloads=True,
output_directories=("gopath", ),
description=f"Download Go module {request.name}@{request.version}.",
),
)
if len(download_result.stdout) == 0:
raise AssertionError(
f"Expected output from `go mod download` for {request.name}@{request.version}."
)
module_metadata = json.loads(download_result.stdout)
module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"],
"gopath/")
go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/")
# Subset the output directory to just the module sources and go.mod (which may be generated).
module_sources_snapshot = await Get(
Snapshot,
DigestSubset(
download_result.output_digest,
PathGlobs(
[f"{module_sources_relpath}/**", go_mod_relpath],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
f"the download of Go module {request.name}@{request.version}",
),
),
)
# Determine directories with potential Go packages in them.
candidate_package_dirs = []
files_by_dir = group_by_dir(p for p in module_sources_snapshot.files
if p.startswith(module_sources_relpath))
for maybe_pkg_dir, files in files_by_dir.items():
# Skip directories where "testdata" would end up in the import path.
# See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585
if "testdata" in maybe_pkg_dir.split("/"):
continue
# Consider directories with at least one `.go` file as package candidates.
if any(f for f in files if f.endswith(".go")):
candidate_package_dirs.append(maybe_pkg_dir)
candidate_package_dirs.sort()
# Analyze all of the packages in this module.
analyzer_relpath = "__analyzer"
analysis_result = await Get(
ProcessResult,
Process(
[
os.path.join(analyzer_relpath, analyzer.path),
*candidate_package_dirs
],
input_digest=module_sources_snapshot.digest,
immutable_input_digests={
analyzer_relpath: analyzer.digest,
},
description=
f"Analyze metadata for Go third-party module: {request.name}@{request.version}",
level=LogLevel.DEBUG,
env={"CGO_ENABLED": "0"},
),
)
if len(analysis_result.stdout) == 0:
return AnalyzedThirdPartyModule(FrozenOrderedSet())
package_analysis_gets = []
for pkg_path, pkg_json in zip(
candidate_package_dirs,
ijson.items(analysis_result.stdout, "", multiple_values=True)):
package_analysis_gets.append(
Get(
FallibleThirdPartyPkgAnalysis,
AnalyzeThirdPartyPackageRequest(
pkg_json=_freeze_json_dict(pkg_json),
module_sources_digest=module_sources_snapshot.digest,
module_sources_path=module_sources_relpath,
module_import_path=request.name,
package_path=pkg_path,
minimum_go_version=request.minimum_go_version,
),
))
analyzed_packages_fallible = await MultiGet(package_analysis_gets)
analyzed_packages = [
pkg.analysis for pkg in analyzed_packages_fallible
if pkg.analysis and pkg.exit_code == 0
]
return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages))
async def generate_targets_from_go_mod(
request: GenerateTargetsFromGoModRequest,
files_not_found_behavior: FilesNotFoundBehavior,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator_addr = request.generator.address
go_mod_info, go_paths = await MultiGet(
Get(GoModInfo, GoModInfoRequest(generator_addr)),
Get(
Paths,
PathGlobs,
request.generator[GoModPackageSourcesField].path_globs(
files_not_found_behavior),
),
)
all_third_party_packages = await Get(
AllThirdPartyPackages,
AllThirdPartyPackagesRequest(go_mod_info.stripped_digest),
)
dir_to_filenames = group_by_dir(go_paths.files)
matched_dirs = [
dir for dir, filenames in dir_to_filenames.items() if filenames
]
def create_first_party_package_tgt(dir: str) -> GoFirstPartyPackageTarget:
subpath = fast_relpath(dir, generator_addr.spec_path)
import_path = f"{go_mod_info.import_path}/{subpath}" if subpath else go_mod_info.import_path
return GoFirstPartyPackageTarget(
{
GoImportPathField.alias:
import_path,
GoFirstPartyPackageSubpathField.alias:
subpath,
GoFirstPartyPackageSourcesField.alias:
tuple(
sorted(
os.path.join(subpath, f)
for f in dir_to_filenames[dir])),
},
# E.g. `src/go:mod#./subdir`.
generator_addr.create_generated(f"./{subpath}"),
union_membership,
residence_dir=dir,
)
first_party_pkgs = (create_first_party_package_tgt(dir)
for dir in matched_dirs)
def create_third_party_package_tgt(
pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: pkg_info.import_path},
# E.g. `src/go:mod#github.com/google/uuid`.
generator_addr.create_generated(pkg_info.import_path),
union_membership,
residence_dir=generator_addr.spec_path,
)
third_party_pkgs = (create_third_party_package_tgt(pkg_info)
for pkg_info in all_third_party_packages.
import_paths_to_pkg_info.values())
return GeneratedTargets(request.generator,
(*first_party_pkgs, *third_party_pkgs))
async def find_putative_go_targets(
request: PutativeGoTargetsRequest, all_owned_sources: AllOwnedSources
) -> PutativeTargets:
putative_targets = []
all_go_mod_files, all_go_files, all_go_files_digest_contents = await MultiGet(
Get(Paths, PathGlobs, request.search_paths.path_globs("go.mod")),
Get(Paths, PathGlobs, request.search_paths.path_globs("*.go")),
Get(DigestContents, PathGlobs, request.search_paths.path_globs("*.go")),
)
# Add `go_mod` targets.
unowned_go_mod_files = set(all_go_mod_files.files) - set(all_owned_sources)
for dirname, filenames in group_by_dir(unowned_go_mod_files).items():
putative_targets.append(
PutativeTarget.for_target_type(
GoModTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
)
)
# Add `go_package` targets.
unowned_go_files = set(all_go_files.files) - set(all_owned_sources)
for dirname, filenames in group_by_dir(unowned_go_files).items():
# Ignore paths that have `testdata` or `vendor` in them.
# From `go help packages`: Note, however, that a directory named vendor that itself contains code
# is not a vendored package: cmd/vendor would be a command named vendor.
dirname_parts = PurePath(dirname).parts
if "testdata" in dirname_parts or "vendor" in dirname_parts[0:-1]:
continue
putative_targets.append(
PutativeTarget.for_target_type(
GoPackageTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
)
)
# Add `go_binary` targets.
main_package_dirs = [
os.path.dirname(file_content.path)
for file_content in all_go_files_digest_contents
if has_package_main(file_content.content)
]
existing_targets = await Get(
UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in main_package_dirs)
)
owned_main_packages = await MultiGet(
Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(t[GoBinaryMainPackageField]))
for t in existing_targets
if t.has_field(GoBinaryMainPackageField)
)
unowned_main_package_dirs = set(main_package_dirs) - {
# NB: We assume the `go_package` lives in the directory it's defined, which we validate
# by e.g. banning `**` in its sources field.
pkg.address.spec_path
for pkg in owned_main_packages
}
putative_targets.extend(
PutativeTarget.for_target_type(
GoBinaryTarget,
path=main_pkg_dir,
name="bin",
triggering_sources=tuple(),
)
for main_pkg_dir in unowned_main_package_dirs
)
return PutativeTargets(putative_targets)
async def find_putative_go_targets(
request: PutativeGoTargetsRequest,
all_owned_sources: AllOwnedSources,
golang_subsystem: GolangSubsystem,
) -> PutativeTargets:
putative_targets = []
_all_go_mod_paths = await Get(Paths, PathGlobs,
request.path_globs("go.mod"))
all_go_mod_files = set(_all_go_mod_paths.files)
all_go_mod_dirs = {os.path.dirname(fp) for fp in all_go_mod_files}
if golang_subsystem.tailor_go_mod_targets:
unowned_go_mod_files = all_go_mod_files - set(all_owned_sources)
for dirname, filenames in group_by_dir(unowned_go_mod_files).items():
putative_targets.append(
PutativeTarget.for_target_type(
GoModTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
))
if golang_subsystem.tailor_package_targets:
all_go_files = await Get(Paths, PathGlobs, request.path_globs("*.go"))
unowned_go_files = set(all_go_files.files) - set(all_owned_sources)
for dirname, filenames in group_by_dir(unowned_go_files).items():
# Ignore paths that have `testdata` or `vendor` in them.
# From `go help packages`: Note, however, that a directory named vendor that itself
# contains code is not a vendored package: cmd/vendor would be a command named vendor.
dirname_parts = PurePath(dirname).parts
if "testdata" in dirname_parts or "vendor" in dirname_parts[0:-1]:
continue
if not has_go_mod_ancestor(dirname, all_go_mod_dirs):
continue
putative_targets.append(
PutativeTarget.for_target_type(
GoPackageTarget,
path=dirname,
name=None,
triggering_sources=sorted(filenames),
))
if golang_subsystem.tailor_binary_targets:
all_go_files_digest_contents = await Get(DigestContents, PathGlobs,
request.path_globs("*.go"))
main_package_dirs = []
for file_content in all_go_files_digest_contents:
dirname = os.path.dirname(file_content.path)
if has_package_main(file_content.content) and has_go_mod_ancestor(
dirname, all_go_mod_dirs):
main_package_dirs.append(dirname)
existing_targets = await Get(
UnexpandedTargets,
RawSpecs(
ancestor_globs=tuple(
AncestorGlobSpec(d) for d in main_package_dirs),
description_of_origin="the `go_binary` tailor rule",
),
)
owned_main_packages = await MultiGet(
Get(GoBinaryMainPackage,
GoBinaryMainPackageRequest(t[GoBinaryMainPackageField]))
for t in existing_targets if t.has_field(GoBinaryMainPackageField))
unowned_main_package_dirs = set(main_package_dirs) - {
# NB: We assume the `go_package` lives in the directory it's defined, which we validate
# by e.g. banning `**` in its sources field.
pkg.address.spec_path
for pkg in owned_main_packages
}
putative_targets.extend(
PutativeTarget.for_target_type(
GoBinaryTarget,
path=main_pkg_dir,
name="bin",
triggering_sources=tuple(),
) for main_pkg_dir in unowned_main_package_dirs)
return PutativeTargets(putative_targets)
async def find_putative_targets(
req: PutativePythonTargetsRequest,
all_owned_sources: AllOwnedSources,
python_setup: PythonSetup,
) -> PutativeTargets:
# Find library/test/test_util targets.
all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py")
all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
classified_unowned_py_files = classify_source_files(unowned_py_files)
pts = []
for tgt_type, paths in classified_unowned_py_files.items():
for dirname, filenames in group_by_dir(paths).items():
if issubclass(tgt_type, PythonTestsGeneratorTarget):
name = "tests"
kwargs = {"name": name}
elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
name = "test_utils"
kwargs = {"name": name}
else:
name = os.path.basename(dirname)
kwargs = {}
if (
python_setup.tailor_ignore_solitary_init_files
and tgt_type == PythonSourcesGeneratorTarget
and filenames == {"__init__.py"}
):
continue
pts.append(
PutativeTarget.for_target_type(
tgt_type, dirname, name, sorted(filenames), kwargs=kwargs
)
)
if python_setup.tailor_requirements_targets:
# Find requirements files.
all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt")
all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs)
unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources)
for req_file in unowned_requirements_files:
path, name = os.path.split(req_file)
pts.append(
PutativeTarget(
path=path,
# python_requirements is a macro and doesn't take a name argument, but the
# PutativeTarget still needs a name for display purposes.
name=name,
type_alias="python_requirements",
triggering_sources=[req_file],
owned_sources=[req_file],
addressable=False,
kwargs={} if name == "requirements.txt" else {"requirements_relpath": name},
)
)
if python_setup.tailor_pex_binary_targets:
# Find binary targets.
# Get all files whose content indicates that they are entry points.
digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs)
entry_points = [
file_content.path
for file_content in digest_contents
if is_entry_point(file_content.content)
]
# Get the modules for these entry points.
src_roots = await Get(
SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)
)
module_to_entry_point = {}
for entry_point in entry_points:
entry_point_path = PurePath(entry_point)
src_root = src_roots.path_to_root[entry_point_path]
stripped_entry_point = entry_point_path.relative_to(src_root.path)
module = PythonModule.create_from_stripped_path(stripped_entry_point)
module_to_entry_point[module.module] = entry_point
# Get existing binary targets for these entry points.
entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points}
possible_existing_binary_targets = await Get(
UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs)
)
possible_existing_binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField]))
for t in possible_existing_binary_targets
if t.has_field(PexEntryPointField)
)
possible_existing_entry_point_modules = {
rep.val.module for rep in possible_existing_binary_entry_points if rep.val
}
unowned_entry_point_modules = (
module_to_entry_point.keys() - possible_existing_entry_point_modules
)
# Generate new targets for entry points that don't already have one.
for entry_point_module in unowned_entry_point_modules:
entry_point = module_to_entry_point[entry_point_module]
path, fname = os.path.split(entry_point)
name = os.path.splitext(fname)[0]
pts.append(
PutativeTarget.for_target_type(
target_type=PexBinary,
path=path,
name=name,
triggering_sources=tuple(),
kwargs={"name": name, "entry_point": fname},
)
)
return PutativeTargets(pts)
async def collect_fixture_configs(
_request: CollectFixtureConfigsRequest,
pytest: PyTest,
python_setup: PythonSetup,
test_extra_env: TestExtraEnv,
targets: Targets,
) -> CollectedJVMLockfileFixtureConfigs:
addresses = [tgt.address for tgt in targets]
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(addresses))
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet(
Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
),
Get(Pex, RequirementsPexRequest(addresses)),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets,
include_files=True,
include_resources=True),
),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(targets),
),
)
script_content = FileContent(path="collect-fixtures.py",
content=COLLECTION_SCRIPT.encode(),
is_executable=True)
script_digest = await Get(Digest, CreateDigest([script_content]))
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=EntryPoint(PurePath(script_content.path).stem),
sources=script_digest,
internal_only=True,
pex_path=[
pytest_pex,
requirements_pex,
],
),
)
config_file_dirs = list(
group_by_dir(prepared_sources.source_files.files).keys())
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(config_file_dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
pytest_config_digest = config_files.snapshot.digest
input_digest = await Get(
Digest,
MergeDigests((
prepared_sources.source_files.snapshot.digest,
pytest_config_digest,
)),
)
extra_env = {
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
}
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=[
name for name in root_sources.source_files.files
if name.endswith(".py")
],
extra_env=extra_env,
input_digest=input_digest,
output_files=("tests.json", ),
description="Collect test lockfile requirements from all tests.",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
result = await Get(ProcessResult, Process, process)
digest_contents = await Get(DigestContents, Digest, result.output_digest)
assert len(digest_contents) == 1
assert digest_contents[0].path == "tests.json"
raw_config_data = json.loads(digest_contents[0].content)
configs = []
for item in raw_config_data:
config = JVMLockfileFixtureConfig(
definition=JVMLockfileFixtureDefinition.from_kwargs(
item["kwargs"]),
test_file_path=item["test_file_path"],
)
configs.append(config)
return CollectedJVMLockfileFixtureConfigs(configs)
