def write_reports(
all_results: tuple[_ResultsWithReports, ...],
workspace: Workspace,
dist_dir: DistDir,
*,
goal_name: str,
get_name: Callable[[_R], str],
) -> None:
disambiguated_dirs: set[str] = set()
def write_report(digest: Digest, subdir: str) -> None:
while subdir in disambiguated_dirs:
# It's unlikely that two distinct partition descriptions will become the
# same after path_safe(), but might as well be safe.
subdir += "_"
disambiguated_dirs.add(subdir)
output_dir = str(dist_dir.relpath / goal_name / subdir)
workspace.write_digest(digest, path_prefix=output_dir)
logger.info(f"Wrote {goal_name} report files to {output_dir}.")
for results in all_results:
tool_name = get_name(results).lower() # type: ignore[arg-type]
if len(results.results
) == 1 and results.results[0].report != EMPTY_DIGEST:
write_report(results.results[0].report, tool_name)
else:
for result in results.results:
if result.report != EMPTY_DIGEST:
write_report(
result.report,
os.path.join(
tool_name,
path_safe(result.partition_description or "all")),
)
def assert_built(rule_runner: RuleRunner, request: BuildGoPackageRequest, *,
expected_import_paths: list[str]) -> None:
built_package = rule_runner.request(BuiltGoPackage, [request])
result_files = rule_runner.request(Snapshot, [built_package.digest]).files
expected = {
import_path: os.path.join("__pkgs__", path_safe(import_path),
"__pkg__.a")
for import_path in expected_import_paths
}
assert dict(built_package.import_paths_to_pkg_a_files) == expected
assert sorted(result_files) == sorted(expected.values())
def test_path_safe() -> None:
assert "abcDEF123" == path_safe("abcDEF123")
assert "CPython>=2.7,<3 (fun times)" == path_safe(
"CPython>=2.7,<3 (fun times)")
assert "foo bar_ baz_" == path_safe("foo bar! baz@")
async def build_go_package(request: BuildGoPackageRequest) -> FallibleBuiltGoPackage:
maybe_built_deps = await MultiGet(
Get(FallibleBuiltGoPackage, BuildGoPackageRequest, build_request)
for build_request in request.direct_dependencies
)
import_paths_to_pkg_a_files: dict[str, str] = {}
dep_digests = []
for maybe_dep in maybe_built_deps:
if maybe_dep.output is None:
return dataclasses.replace(
maybe_dep, import_path=request.import_path, dependency_failed=True
)
dep = maybe_dep.output
import_paths_to_pkg_a_files.update(dep.import_paths_to_pkg_a_files)
dep_digests.append(dep.digest)
merged_deps_digest, import_config, embedcfg = await MultiGet(
Get(Digest, MergeDigests(dep_digests)),
Get(ImportConfig, ImportConfigRequest(FrozenDict(import_paths_to_pkg_a_files))),
Get(RenderedEmbedConfig, RenderEmbedConfigRequest(request.embed_config)),
)
input_digest = await Get(
Digest,
MergeDigests([merged_deps_digest, import_config.digest, embedcfg.digest, request.digest]),
)
assembly_digests = None
symabis_path = None
if request.s_file_names:
assembly_setup = await Get(
FallibleAssemblyPreCompilation,
AssemblyPreCompilationRequest(input_digest, request.s_file_names, request.subpath),
)
if assembly_setup.result is None:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_setup.exit_code,
stdout=assembly_setup.stdout,
stderr=assembly_setup.stderr,
)
input_digest = assembly_setup.result.merged_compilation_input_digest
assembly_digests = assembly_setup.result.assembly_digests
symabis_path = "./symabis"
compile_args = [
"tool",
"compile",
"-o",
"__pkg__.a",
"-pack",
"-p",
request.import_path,
"-importcfg",
import_config.CONFIG_PATH,
# See https://github.com/golang/go/blob/f229e7031a6efb2f23241b5da000c3b3203081d6/src/cmd/go/internal/work/gc.go#L79-L100
# for why Go sets the default to 1.16.
f"-lang=go{request.minimum_go_version or '1.16'}",
]
if symabis_path:
compile_args.extend(["-symabis", symabis_path])
if embedcfg.digest != EMPTY_DIGEST:
compile_args.extend(["-embedcfg", RenderedEmbedConfig.PATH])
if not request.s_file_names:
# If there are no non-Go sources, then pass -complete flag which tells the compiler that the provided
# Go files are the entire package.
compile_args.append("-complete")
relativized_sources = (
f"./{request.subpath}/{name}" if request.subpath else f"./{name}"
for name in request.go_file_names
)
compile_args.extend(["--", *relativized_sources])
compile_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=tuple(compile_args),
description=f"Compile Go package: {request.import_path}",
output_files=("__pkg__.a",),
),
)
if compile_result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
compile_result.exit_code,
stdout=compile_result.stdout.decode("utf-8"),
stderr=compile_result.stderr.decode("utf-8"),
)
compilation_digest = compile_result.output_digest
if assembly_digests:
assembly_result = await Get(
AssemblyPostCompilation,
AssemblyPostCompilationRequest(
compilation_digest,
assembly_digests,
request.s_file_names,
request.subpath,
),
)
if assembly_result.result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_result.result.exit_code,
stdout=assembly_result.result.stdout.decode("utf-8"),
stderr=assembly_result.result.stderr.decode("utf-8"),
)
assert assembly_result.merged_output_digest
compilation_digest = assembly_result.merged_output_digest
path_prefix = os.path.join("__pkgs__", path_safe(request.import_path))
import_paths_to_pkg_a_files[request.import_path] = os.path.join(path_prefix, "__pkg__.a")
output_digest = await Get(Digest, AddPrefix(compilation_digest, path_prefix))
merged_result_digest = await Get(Digest, MergeDigests([*dep_digests, output_digest]))
output = BuiltGoPackage(merged_result_digest, FrozenDict(import_paths_to_pkg_a_files))
return FallibleBuiltGoPackage(output, request.import_path)
async def export_virtualenv(request: _ExportVenvRequest,
python_setup: PythonSetup,
pex_pex: PexPEX) -> ExportResult:
if request.resolve:
interpreter_constraints = InterpreterConstraints(
python_setup.resolves_to_interpreter_constraints.get(
request.resolve, python_setup.interpreter_constraints))
else:
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.root_python_targets,
python_setup) or InterpreterConstraints(
python_setup.interpreter_constraints)
min_interpreter = interpreter_constraints.snap_to_minimum(
python_setup.interpreter_universe)
if not min_interpreter:
err_msg = ((
f"The resolve '{request.resolve}' (from `[python].resolves`) has invalid interpreter "
f"constraints, which are set via `[python].resolves_to_interpreter_constraints`: "
f"{interpreter_constraints}. Could not determine the minimum compatible interpreter."
) if request.resolve else (
"The following interpreter constraints were computed for all the targets for which "
f"export was requested: {interpreter_constraints}. There is no python interpreter "
"compatible with these constraints. Please restrict the target set to one that shares "
"a compatible interpreter."))
raise ExportError(err_msg)
requirements_pex = await Get(
Pex,
RequirementsPexRequest(
(tgt.address for tgt in request.root_python_targets),
internal_only=True,
hardcoded_interpreter_constraints=min_interpreter,
),
)
# Get the full python version (including patch #), so we can use it as the venv name.
res = await Get(
ProcessResult,
PexProcess(
pex=requirements_pex,
description="Get interpreter version",
argv=[
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"
],
),
)
py_version = res.stdout.strip().decode()
dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve))
if request.resolve else os.path.join("python", "virtualenv"))
merged_digest = await Get(
Digest, MergeDigests([pex_pex.digest, requirements_pex.digest]))
pex_pex_path = os.path.join("{digest_root}", pex_pex.exe)
return ExportResult(
f"virtualenv for the resolve '{request.resolve}' (using {min_interpreter})",
dest,
digest=merged_digest,
post_processing_cmds=[
PostProcessingCommand(
[
pex_pex_path,
os.path.join("{digest_root}", requirements_pex.name),
"venv",
"--pip",
"--collisions-ok",
"--remove=all",
f"{{digest_root}}/{py_version}",
],
{"PEX_MODULE": "pex.tools"},
),
PostProcessingCommand(["rm", "-f", pex_pex_path]),
],
)
async def export_virtualenv(request: _ExportVenvRequest,
python_setup: PythonSetup,
pex_pex: PexPEX) -> ExportResult:
if request.resolve:
interpreter_constraints = InterpreterConstraints(
python_setup.resolves_to_interpreter_constraints.get(
request.resolve, python_setup.interpreter_constraints))
else:
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.root_python_targets,
python_setup) or InterpreterConstraints(
python_setup.interpreter_constraints)
requirements_pex = await Get(
Pex,
RequirementsPexRequest(
(tgt.address for tgt in request.root_python_targets),
hardcoded_interpreter_constraints=interpreter_constraints,
),
)
# Note that an internal-only pex will always have the `python` field set.
# See the build_pex() rule in pex.py.
interpreter = cast(PythonExecutable, requirements_pex.python)
# Get the full python version (including patch #), so we can use it as the venv name.
res = await Get(
ProcessResult,
Process(
description="Get interpreter version",
argv=[
interpreter.path,
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))",
],
),
)
py_version = res.stdout.strip().decode()
dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve))
if request.resolve else os.path.join("python", "virtualenv"))
merged_digest = await Get(
Digest, MergeDigests([pex_pex.digest, requirements_pex.digest]))
pex_pex_path = os.path.join("{digest_root}", pex_pex.exe)
maybe_resolve_str = f"for the resolve '{request.resolve}' " if request.resolve else ""
return ExportResult(
f"virtualenv {maybe_resolve_str}(using Python {py_version})",
dest,
digest=merged_digest,
post_processing_cmds=[
PostProcessingCommand(
[
interpreter.path,
pex_pex_path,
os.path.join("{digest_root}", requirements_pex.name),
"venv",
"--pip",
"--collisions-ok",
"--remove=all",
f"{{digest_root}}/{py_version}",
],
{"PEX_MODULE": "pex.tools"},
),
PostProcessingCommand(["rm", "-f", pex_pex_path]),
],
)
