def write_reports( all_results: tuple[_ResultsWithReports, ...], workspace: Workspace, dist_dir: DistDir, *, goal_name: str, get_name: Callable[[_R], str], ) -> None: disambiguated_dirs: set[str] = set() def write_report(digest: Digest, subdir: str) -> None: while subdir in disambiguated_dirs: # It's unlikely that two distinct partition descriptions will become the # same after path_safe(), but might as well be safe. subdir += "_" disambiguated_dirs.add(subdir) output_dir = str(dist_dir.relpath / goal_name / subdir) workspace.write_digest(digest, path_prefix=output_dir) logger.info(f"Wrote {goal_name} report files to {output_dir}.") for results in all_results: tool_name = get_name(results).lower() # type: ignore[arg-type] if len(results.results ) == 1 and results.results[0].report != EMPTY_DIGEST: write_report(results.results[0].report, tool_name) else: for result in results.results: if result.report != EMPTY_DIGEST: write_report( result.report, os.path.join( tool_name, path_safe(result.partition_description or "all")), )
def assert_built(rule_runner: RuleRunner, request: BuildGoPackageRequest, *, expected_import_paths: list[str]) -> None: built_package = rule_runner.request(BuiltGoPackage, [request]) result_files = rule_runner.request(Snapshot, [built_package.digest]).files expected = { import_path: os.path.join("__pkgs__", path_safe(import_path), "__pkg__.a") for import_path in expected_import_paths } assert dict(built_package.import_paths_to_pkg_a_files) == expected assert sorted(result_files) == sorted(expected.values())
def test_path_safe() -> None: assert "abcDEF123" == path_safe("abcDEF123") assert "CPython>=2.7,<3 (fun times)" == path_safe( "CPython>=2.7,<3 (fun times)") assert "foo bar_ baz_" == path_safe("foo bar! baz@")
async def build_go_package(request: BuildGoPackageRequest) -> FallibleBuiltGoPackage: maybe_built_deps = await MultiGet( Get(FallibleBuiltGoPackage, BuildGoPackageRequest, build_request) for build_request in request.direct_dependencies ) import_paths_to_pkg_a_files: dict[str, str] = {} dep_digests = [] for maybe_dep in maybe_built_deps: if maybe_dep.output is None: return dataclasses.replace( maybe_dep, import_path=request.import_path, dependency_failed=True ) dep = maybe_dep.output import_paths_to_pkg_a_files.update(dep.import_paths_to_pkg_a_files) dep_digests.append(dep.digest) merged_deps_digest, import_config, embedcfg = await MultiGet( Get(Digest, MergeDigests(dep_digests)), Get(ImportConfig, ImportConfigRequest(FrozenDict(import_paths_to_pkg_a_files))), Get(RenderedEmbedConfig, RenderEmbedConfigRequest(request.embed_config)), ) input_digest = await Get( Digest, MergeDigests([merged_deps_digest, import_config.digest, embedcfg.digest, request.digest]), ) assembly_digests = None symabis_path = None if request.s_file_names: assembly_setup = await Get( FallibleAssemblyPreCompilation, AssemblyPreCompilationRequest(input_digest, request.s_file_names, request.subpath), ) if assembly_setup.result is None: return FallibleBuiltGoPackage( None, request.import_path, assembly_setup.exit_code, stdout=assembly_setup.stdout, stderr=assembly_setup.stderr, ) input_digest = assembly_setup.result.merged_compilation_input_digest assembly_digests = assembly_setup.result.assembly_digests symabis_path = "./symabis" compile_args = [ "tool", "compile", "-o", "__pkg__.a", "-pack", "-p", request.import_path, "-importcfg", import_config.CONFIG_PATH, # See https://github.com/golang/go/blob/f229e7031a6efb2f23241b5da000c3b3203081d6/src/cmd/go/internal/work/gc.go#L79-L100 # for why Go sets the default to 1.16. f"-lang=go{request.minimum_go_version or '1.16'}", ] if symabis_path: compile_args.extend(["-symabis", symabis_path]) if embedcfg.digest != EMPTY_DIGEST: compile_args.extend(["-embedcfg", RenderedEmbedConfig.PATH]) if not request.s_file_names: # If there are no non-Go sources, then pass -complete flag which tells the compiler that the provided # Go files are the entire package. compile_args.append("-complete") relativized_sources = ( f"./{request.subpath}/{name}" if request.subpath else f"./{name}" for name in request.go_file_names ) compile_args.extend(["--", *relativized_sources]) compile_result = await Get( FallibleProcessResult, GoSdkProcess( input_digest=input_digest, command=tuple(compile_args), description=f"Compile Go package: {request.import_path}", output_files=("__pkg__.a",), ), ) if compile_result.exit_code != 0: return FallibleBuiltGoPackage( None, request.import_path, compile_result.exit_code, stdout=compile_result.stdout.decode("utf-8"), stderr=compile_result.stderr.decode("utf-8"), ) compilation_digest = compile_result.output_digest if assembly_digests: assembly_result = await Get( AssemblyPostCompilation, AssemblyPostCompilationRequest( compilation_digest, assembly_digests, request.s_file_names, request.subpath, ), ) if assembly_result.result.exit_code != 0: return FallibleBuiltGoPackage( None, request.import_path, assembly_result.result.exit_code, stdout=assembly_result.result.stdout.decode("utf-8"), stderr=assembly_result.result.stderr.decode("utf-8"), ) assert assembly_result.merged_output_digest compilation_digest = assembly_result.merged_output_digest path_prefix = os.path.join("__pkgs__", path_safe(request.import_path)) import_paths_to_pkg_a_files[request.import_path] = os.path.join(path_prefix, "__pkg__.a") output_digest = await Get(Digest, AddPrefix(compilation_digest, path_prefix)) merged_result_digest = await Get(Digest, MergeDigests([*dep_digests, output_digest])) output = BuiltGoPackage(merged_result_digest, FrozenDict(import_paths_to_pkg_a_files)) return FallibleBuiltGoPackage(output, request.import_path)
async def export_virtualenv(request: _ExportVenvRequest, python_setup: PythonSetup, pex_pex: PexPEX) -> ExportResult: if request.resolve: interpreter_constraints = InterpreterConstraints( python_setup.resolves_to_interpreter_constraints.get( request.resolve, python_setup.interpreter_constraints)) else: interpreter_constraints = InterpreterConstraints.create_from_targets( request.root_python_targets, python_setup) or InterpreterConstraints( python_setup.interpreter_constraints) min_interpreter = interpreter_constraints.snap_to_minimum( python_setup.interpreter_universe) if not min_interpreter: err_msg = (( f"The resolve '{request.resolve}' (from `[python].resolves`) has invalid interpreter " f"constraints, which are set via `[python].resolves_to_interpreter_constraints`: " f"{interpreter_constraints}. Could not determine the minimum compatible interpreter." ) if request.resolve else ( "The following interpreter constraints were computed for all the targets for which " f"export was requested: {interpreter_constraints}. There is no python interpreter " "compatible with these constraints. Please restrict the target set to one that shares " "a compatible interpreter.")) raise ExportError(err_msg) requirements_pex = await Get( Pex, RequirementsPexRequest( (tgt.address for tgt in request.root_python_targets), internal_only=True, hardcoded_interpreter_constraints=min_interpreter, ), ) # Get the full python version (including patch #), so we can use it as the venv name. res = await Get( ProcessResult, PexProcess( pex=requirements_pex, description="Get interpreter version", argv=[ "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))" ], ), ) py_version = res.stdout.strip().decode() dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve)) if request.resolve else os.path.join("python", "virtualenv")) merged_digest = await Get( Digest, MergeDigests([pex_pex.digest, requirements_pex.digest])) pex_pex_path = os.path.join("{digest_root}", pex_pex.exe) return ExportResult( f"virtualenv for the resolve '{request.resolve}' (using {min_interpreter})", dest, digest=merged_digest, post_processing_cmds=[ PostProcessingCommand( [ pex_pex_path, os.path.join("{digest_root}", requirements_pex.name), "venv", "--pip", "--collisions-ok", "--remove=all", f"{{digest_root}}/{py_version}", ], {"PEX_MODULE": "pex.tools"}, ), PostProcessingCommand(["rm", "-f", pex_pex_path]), ], )
async def export_virtualenv(request: _ExportVenvRequest, python_setup: PythonSetup, pex_pex: PexPEX) -> ExportResult: if request.resolve: interpreter_constraints = InterpreterConstraints( python_setup.resolves_to_interpreter_constraints.get( request.resolve, python_setup.interpreter_constraints)) else: interpreter_constraints = InterpreterConstraints.create_from_targets( request.root_python_targets, python_setup) or InterpreterConstraints( python_setup.interpreter_constraints) requirements_pex = await Get( Pex, RequirementsPexRequest( (tgt.address for tgt in request.root_python_targets), hardcoded_interpreter_constraints=interpreter_constraints, ), ) # Note that an internal-only pex will always have the `python` field set. # See the build_pex() rule in pex.py. interpreter = cast(PythonExecutable, requirements_pex.python) # Get the full python version (including patch #), so we can use it as the venv name. res = await Get( ProcessResult, Process( description="Get interpreter version", argv=[ interpreter.path, "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))", ], ), ) py_version = res.stdout.strip().decode() dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve)) if request.resolve else os.path.join("python", "virtualenv")) merged_digest = await Get( Digest, MergeDigests([pex_pex.digest, requirements_pex.digest])) pex_pex_path = os.path.join("{digest_root}", pex_pex.exe) maybe_resolve_str = f"for the resolve '{request.resolve}' " if request.resolve else "" return ExportResult( f"virtualenv {maybe_resolve_str}(using Python {py_version})", dest, digest=merged_digest, post_processing_cmds=[ PostProcessingCommand( [ interpreter.path, pex_pex_path, os.path.join("{digest_root}", requirements_pex.name), "venv", "--pip", "--collisions-ok", "--remove=all", f"{{digest_root}}/{py_version}", ], {"PEX_MODULE": "pex.tools"}, ), PostProcessingCommand(["rm", "-f", pex_pex_path]), ], )