def test_pex_execution(self) -> None:
sources_content = InputFilesContent((
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py", content=b'print("from sub")'),
))
sources = self.request_single_product(Digest, sources_content)
pex_output = self.create_pex_and_get_all_data(entry_point="main",
sources=sources)
pex_files = pex_output["files"]
assert "pex" not in pex_files
assert "main.py" in pex_files
assert "subdir/sub.py" in pex_files
init_subsystem(PythonSetup)
python_setup = PythonSetup.global_instance()
env = {
"PATH": create_path_env_var(python_setup.interpreter_search_paths)
}
process = Process(
argv=("python", "test.pex"),
env=env,
input_files=pex_output["pex"].directory_digest,
description="Run the pex and make sure it works",
)
result = self.request_single_product(ProcessResult, process)
assert result.stdout == b"from main\n"
def test_write_file(self):
request = Process(
argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"),
description="echo roland",
output_files=("roland", ),
input_files=EMPTY_DIRECTORY_DIGEST,
)
process_result = self.request_single_product(ProcessResult, request)
self.assertEqual(
process_result.output_directory_digest,
Digest(
fingerprint=
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16",
serialized_bytes_length=80,
),
)
files_content_result = self.request_single_product(
FilesContent,
process_result.output_directory_digest,
)
self.assertEqual(files_content_result.dependencies,
(FileContent("roland", b"European Burmese", False), ))
def test_create_from_snapshot_with_env(self):
req = Process(
argv=("foo", ),
description="Some process",
env={"VAR": "VAL"},
input_files=EMPTY_DIRECTORY_DIGEST,
)
self.assertEqual(req.env, ("VAR", "VAL"))
def test_fallible_failing_command_returns_exited_result(self):
request = Process(
argv=("/bin/bash", "-c", "exit 1"),
description="one-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
result = self.request_single_product(FallibleProcessResult, request)
self.assertEqual(result.exit_code, 1)
def test_non_fallible_failing_command_raises(self):
request = Process(
argv=("/bin/bash", "-c", "exit 1"),
description="one-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(ProcessResult, request)
self.assertIn("process 'one-cat' failed with exit code 1.",
str(cm.exception))
async def cat_files_process_result_concatted(
cat_exe_req: CatExecutionRequest) -> Concatted:
cat_bin = cat_exe_req.shell_cat
cat_files_snapshot = await Get[Snapshot](PathGlobs, cat_exe_req.path_globs)
process = Process(
argv=cat_bin.argv_from_snapshot(cat_files_snapshot),
input_files=cat_files_snapshot.directory_digest,
description="cat some files",
)
cat_process_result = await Get[ProcessResult](Process, process)
return Concatted(cat_process_result.stdout.decode())
def test_jdk(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
request = Process(
argv=("/bin/cat", ".jdk/roland"),
input_files=EMPTY_DIRECTORY_DIGEST,
description="cat JDK roland",
jdk_home=temp_dir,
)
result = self.request_single_product(ProcessResult, request)
self.assertEqual(result.stdout, b"European Burmese")
def test_timeout(self):
request = Process(
argv=("/bin/bash", "-c",
"/bin/sleep 0.2; /bin/echo -n 'European Burmese'"),
timeout_seconds=0.1,
description="sleepy-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
result = self.request_single_product(FallibleProcessResult, request)
self.assertNotEqual(result.exit_code, 0)
self.assertIn(b"Exceeded timeout", result.stdout)
self.assertIn(b"sleepy-cat", result.stdout)
def test_platform_on_local_epr_result(self) -> None:
this_platform = Platform.current
req = Process(
argv=("/bin/echo", "test"),
input_files=EMPTY_DIRECTORY_DIGEST,
description="Run some program that will exit cleanly.",
)
result = self.request_single_product(FallibleProcessResultWithPlatform, req)
assert result.exit_code == 0
assert result.platform == this_platform
async def get_javac_version_output(
javac_version_command: JavacVersionExecutionRequest,
) -> JavacVersionOutput:
javac_version_proc_req = Process(
argv=javac_version_command.gen_argv(),
description=javac_version_command.description,
input_files=EMPTY_DIRECTORY_DIGEST,
)
javac_version_proc_result = await Get[ProcessResult](
Process,
javac_version_proc_req,
)
return JavacVersionOutput(javac_version_proc_result.stderr.decode())
def create_execute_request(
self,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
*,
pex_path: str,
pex_args: Iterable[str],
description: str,
input_files: Digest,
env: Optional[Mapping[str, str]] = None,
**kwargs: Any) -> Process:
"""Creates an Process that will run a PEX hermetically.
:param python_setup: The parameters for selecting python interpreters to use when invoking
the PEX.
:param subprocess_encoding_environment: The locale settings to use for the PEX invocation.
:param pex_path: The path within `input_files` of the PEX file (or directory if a loose
pex).
:param pex_args: The arguments to pass to the PEX executable.
:param description: A description of the process execution to be performed.
:param input_files: The files that contain the pex itself and any input files it needs to
run against.
:param env: The environment to run the PEX in.
:param **kwargs: Any additional :class:`Process` kwargs to pass through.
"""
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
argv = ("python", pex_path, *pex_args)
hermetic_env = dict(
PATH=create_path_env_var(python_setup.interpreter_search_paths),
PEX_ROOT="./pex_root",
PEX_INHERIT_PATH="false",
PEX_IGNORE_RCFILES="true",
**subprocess_encoding_environment.invocation_environment_dict)
if env:
hermetic_env.update(env)
return Process(argv=argv,
input_files=input_files,
description=description,
env=hermetic_env,
**kwargs)
def test_multiple_file_creation(self):
input_files_content = InputFilesContent((
FileContent(path="a.txt", content=b"hello"),
FileContent(path="b.txt", content=b"goodbye"),
))
digest = self.request_single_product(Digest, input_files_content)
req = Process(
argv=("/bin/cat", "a.txt", "b.txt"),
input_files=digest,
description="cat the contents of this file",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, b"hellogoodbye")
def test_input_file_creation(self):
file_name = "some.filename"
file_contents = b"some file contents"
input_file = InputFilesContent((FileContent(path=file_name,
content=file_contents), ))
digest = self.request_single_product(Digest, input_file)
req = Process(
argv=("/bin/cat", file_name),
input_files=digest,
description="cat the contents of this file",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, file_contents)
def test_not_executable(self):
file_name = "echo.sh"
file_contents = b'#!/bin/bash -eu\necho "Hello"\n'
input_file = InputFilesContent((FileContent(path=file_name,
content=file_contents), ))
digest = self.request_single_product(Digest, input_file)
req = Process(
argv=("./echo.sh", ),
input_files=digest,
description="cat the contents of this file",
)
with self.assertRaisesWithMessageContaining(ExecutionError,
"Permission"):
self.request_single_product(ProcessResult, req)
def test_file_in_directory_creation(self):
path = "somedir/filename"
content = b"file contents"
input_file = InputFilesContent((FileContent(path=path,
content=content), ))
digest = self.request_single_product(Digest, input_file)
req = Process(
argv=("/bin/cat", "somedir/filename"),
input_files=digest,
description=
"Cat a file in a directory to make sure that doesn't break",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, content)
def test_executable(self):
file_name = "echo.sh"
file_contents = b'#!/bin/bash -eu\necho "Hello"\n'
input_file = InputFilesContent((FileContent(path=file_name,
content=file_contents,
is_executable=True), ))
digest = self.request_single_product(Digest, input_file)
req = Process(
argv=("./echo.sh", ),
input_files=digest,
description="cat the contents of this file",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, b"Hello\n")
def _run_bootstrapper(self, bridge_jar, context):
bootstrapper_entry = self._zinc_factory._compiler_bootstrapper(self._products)
bootstrapper = self._relative_to_buildroot(bootstrapper_entry.path)
# CLI args and their associated ClasspathEntry objects.
bootstrap_cp_entries = (
("--compiler-interface", self._compiler_interface),
("--compiler-bridge-src", self._compiler_bridge),
("--scala-compiler", self.scala_compiler),
("--scala-library", self.scala_library),
("--scala-reflect", self.scala_reflect),
)
bootstrapper_args = [
"--out",
self._relative_to_buildroot(bridge_jar),
]
for arg, cp_entry in bootstrap_cp_entries:
bootstrapper_args.append(arg)
bootstrapper_args.append(self._relative_to_buildroot(cp_entry.path))
inputs_digest = context._scheduler.merge_directories(
[bootstrapper_entry.directory_digest]
+ [entry.directory_digest for _, entry in bootstrap_cp_entries]
)
argv = tuple(
[".jdk/bin/java"]
+ ["-cp", bootstrapper, Zinc.ZINC_BOOTSTRAPER_MAIN]
+ bootstrapper_args
)
req = Process(
argv=argv,
input_files=inputs_digest,
output_files=(self._relative_to_buildroot(bridge_jar),),
description="bootstrap compiler bridge.",
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=self.underlying_dist.home,
)
return context.execute_process_synchronously_or_raise(
req, "zinc-subsystem", [WorkUnitLabel.COMPILER]
)
def _execute_hermetic_compile(self, cmd, ctx):
# For now, executing a compile remotely only works for targets that
# do not have any dependencies or inner classes
input_snapshot = ctx.target.sources_snapshot(
scheduler=self.context._scheduler)
output_files = tuple(
# Assume no extra .class files to grab. We'll fix up that case soon.
# Drop the source_root from the file path.
# Assumes `-d .` has been put in the command.
os.path.relpath(f.replace(".java", ".class"),
ctx.target.target_base)
for f in input_snapshot.files if f.endswith(".java"))
process = Process(
argv=tuple(cmd),
input_files=input_snapshot.directory_digest,
output_files=output_files,
description=f"Compiling {ctx.target.address.spec} with javac",
)
exec_result = self.context.execute_process_synchronously_without_raising(
process,
"javac",
(WorkUnitLabel.TASK, WorkUnitLabel.JVM),
)
# Dump the output to the .pants.d directory where it's expected by downstream tasks.
merged_directories = self.context._scheduler.merge_directories([
exec_result.output_directory_digest,
self.post_compile_extra_resources_digest(
ctx, prepend_post_merge_relative_path=False),
])
classes_directory = Path(ctx.classes_dir.path).relative_to(
get_buildroot())
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(merged_directories,
path_prefix=str(classes_directory)), )
async def javac_compile_process_result(
javac_compile_req: JavacCompileRequest, ) -> JavacCompileResult:
java_files = javac_compile_req.javac_sources.java_files
for java_file in java_files:
if not java_file.endswith(".java"):
raise ValueError(
f"Can only compile .java files but got {java_file}")
sources_snapshot = await Get[Snapshot](PathGlobs, PathGlobs(java_files))
output_dirs = tuple(
{os.path.dirname(java_file)
for java_file in java_files})
process = Process(
argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
input_files=sources_snapshot.directory_digest,
output_directories=output_dirs,
description="javac compilation",
)
javac_proc_result = await Get[ProcessResult](Process, process)
return JavacCompileResult(
javac_proc_result.stdout.decode(),
javac_proc_result.stderr.decode(),
javac_proc_result.output_directory_digest,
)
def _compile_hermetic(
self,
jvm_options,
ctx,
classes_dir,
jar_file,
compiler_bridge_classpath_entry,
dependency_classpath,
scalac_classpath_entries,
):
zinc_relpath = fast_relpath(self._zinc.zinc.path, get_buildroot())
snapshots = [
ctx.target.sources_snapshot(self.context._scheduler),
]
# scala_library() targets with java_sources have circular dependencies on those java source
# files, and we provide them to the same zinc command line that compiles the scala, so we need
# to make sure those source files are available in the hermetic execution sandbox.
java_sources_targets = getattr(ctx.target, "java_sources", [])
java_sources_snapshots = [
tgt.sources_snapshot(self.context._scheduler)
for tgt in java_sources_targets
]
snapshots.extend(java_sources_snapshots)
# Ensure the dependencies and compiler bridge jars are available in the execution sandbox.
relevant_classpath_entries = dependency_classpath + [
compiler_bridge_classpath_entry,
self._nailgun_server_classpath_entry(
), # We include nailgun-server, to use it to start servers when needed from the hermetic execution case.
]
directory_digests = [
entry.directory_digest for entry in relevant_classpath_entries
if entry.directory_digest
]
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if not dep.directory_digest:
raise AssertionError(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution of zinc".format(dep))
directory_digests.extend(
classpath_entry.directory_digest
for classpath_entry in scalac_classpath_entries)
if self._zinc.use_native_image:
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._zinc.native_image(
self.context)
native_image_snapshots = [
native_image_snapshot.directory_digest,
]
scala_boot_classpath = [
classpath_entry.path
for classpath_entry in scalac_classpath_entries
] + [
# We include rt.jar on the scala boot classpath because the compiler usually gets its
# contents from the VM it is executing in, but not in the case of a native image. This
# resolves a `object java.lang.Object in compiler mirror not found.` error.
".jdk/jre/lib/rt.jar",
# The same goes for the jce.jar, which provides javax.crypto.
".jdk/jre/lib/jce.jar",
]
image_specific_argv = [
native_image_path,
"-java-home",
".jdk",
f"-Dscala.boot.class.path={os.pathsep.join(scala_boot_classpath)}",
"-Dscala.usejavacp=true",
]
else:
native_image_snapshots = []
# TODO: Lean on distribution for the bin/java appending here
image_specific_argv = (
[".jdk/bin/java"] + jvm_options +
["-cp", zinc_relpath, Zinc.ZINC_COMPILE_MAIN])
(argfile_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
relpath_to_analysis = fast_relpath(ctx.analysis_file, get_buildroot())
merged_local_only_scratch_inputs = self._compute_local_only_inputs(
classes_dir, relpath_to_analysis, jar_file)
# TODO: Extract something common from Executor._create_command to make the command line
argv = image_specific_argv + [f"@{argfile_snapshot.files[0]}"]
merged_input_digest = self.context._scheduler.merge_directories(
[self._zinc.zinc.directory_digest] +
[s.directory_digest for s in snapshots] + directory_digests +
native_image_snapshots + [
self.post_compile_extra_resources_digest(ctx),
argfile_snapshot.directory_digest
])
# NB: We always capture the output jar, but if classpath jars are not used, we additionally
# capture loose classes from the workspace. This is because we need to both:
# 1) allow loose classes as an input to dependent compiles
# 2) allow jars to be materialized at the end of the run.
output_directories = () if self.get_options().use_classpath_jars else (
classes_dir, )
req = Process(
argv=tuple(argv),
input_files=merged_input_digest,
output_files=(jar_file, relpath_to_analysis),
output_directories=output_directories,
description=f"zinc compile for {ctx.target.address.spec}",
unsafe_local_only_files_because_we_favor_speed_over_correctness_for_this_rule
=merged_local_only_scratch_inputs,
jdk_home=self._zinc.underlying_dist.home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def console_output(self, targets):
input_snapshots = tuple(
target.sources_snapshot(scheduler=self.context._scheduler)
for target in targets)
input_files = {
f
for snapshot in input_snapshots for f in snapshot.files
}
# TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
with temporary_dir() as tmpdir:
list_file = os.path.join(tmpdir, "input_files_list")
with open(list_file, "w") as list_file_out:
for input_file in sorted(input_files):
list_file_out.write(input_file)
list_file_out.write("\n")
list_file_snapshot = self.context._scheduler.capture_snapshots(
(PathGlobsAndRoot(
PathGlobs(("input_files_list", )),
tmpdir,
), ))[0]
cloc_path, cloc_snapshot = ClocBinary.global_instance(
).hackily_snapshot(self.context)
directory_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in input_snapshots + (
cloc_snapshot,
list_file_snapshot,
)))
cmd = (
"/usr/bin/perl",
cloc_path,
"--skip-uniqueness",
"--ignored=ignored",
"--list-file=input_files_list",
"--report-file=report",
)
# The cloc script reaches into $PATH to look up perl. Let's assume it's in /usr/bin.
req = Process(
argv=cmd,
input_files=directory_digest,
output_files=("ignored", "report"),
description="cloc",
)
exec_result = self.context.execute_process_synchronously_or_raise(
req, "cloc", (WorkUnitLabel.TOOL, ))
files_content_tuple = self.context._scheduler.product_request(
FilesContent,
[exec_result.output_directory_digest])[0].dependencies
files_content = {
fc.path: fc.content.decode()
for fc in files_content_tuple
}
for line in files_content["report"].split("\n"):
yield line
if self.get_options().ignored:
yield "Ignored the following files:"
for line in files_content["ignored"].split("\n"):
yield line
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
use_youtline = tool_name == "scalac-outliner"
tool_classpath_abs = self._scalac_classpath if use_youtline else self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
rsc_jvm_options = Rsc.global_instance().get_options().jvm_options
if not use_youtline and self._rsc.use_native_image:
if rsc_jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(rsc_jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = (
[distribution.java] + rsc_jvm_options +
["-cp", os.pathsep.join(tool_classpath), main])
(argfile_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + [f"@{argfile_snapshot.files[0]}"]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.directory_digest for s in additional_snapshots) +
(argfile_snapshot.directory_digest, ))
epr = Process(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description=f"run {tool_name} for {ctx.target}",
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# Process requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_directory_digest.dump(ctx.rsc_jar_file.path)
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest), )
ctx.rsc_jar_file.hydrate_missing_directory_digest(
res.output_directory_digest)
return res