async def run_bash_binary(field_set: BashRunFieldSet,
bash_program: BashProgram,
bash_setup: BashSetup) -> RunRequest:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
# We need to include all relevant transitive dependencies in the environment. We also get the
# binary's sources so that we know the script name. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-and-target-api.
binary_sources_request = Get(SourceFiles,
SourceFilesRequest([field_set.sources]))
all_sources_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(BashSources, FilesSources, ResourcesSources),
),
)
binary_sources, all_sources = await MultiGet(binary_sources_request,
all_sources_request)
# We join the relative path to our program with the template string "{chroot}", which will get
# substituted with the path to the temporary directory where our program runs. This ensures
# that we run the correct file.
# Note that `BashBinarySources` will have already validated that there is exactly one file in
# the sources field.
script_name = os.path.join("{chroot}", binary_sources.files[0])
return RunRequest(
digest=all_sources.snapshot.digest,
args=[bash_program.exe, script_name],
extra_env=bash_setup.env_dict,
)
async def create_go_binary_run_request(
field_set: GoBinaryFieldSet) -> RunRequest:
binary = await Get(BuiltPackage, PackageFieldSet, field_set)
artifact_relpath = binary.artifacts[0].relpath
assert artifact_relpath is not None
return RunRequest(digest=binary.digest,
args=(os.path.join("{chroot}", artifact_relpath), ))
async def run_shell_command_request(shell_command: RunShellCommand) -> RunRequest:
wrapped_tgt = await Get(WrappedTarget, Address, shell_command.address)
process = await Get(Process, ShellCommandProcessRequest(wrapped_tgt.target))
return RunRequest(
digest=process.input_digest,
args=process.argv,
extra_env=process.env,
)
def create_mock_run_request(self, program_text: bytes) -> RunRequest:
digest = self.request_single_product(
Digest,
CreateDigest([
FileContent(path="program.py",
content=program_text,
is_executable=True)
]),
)
return RunRequest(digest=digest, binary_name="program.py")
def create_mock_run_request(self, program_text: bytes) -> RunRequest:
digest = self.request_single_product(
Digest,
CreateDigest([
FileContent(path="program.py",
content=program_text,
is_executable=True)
]),
)
return RunRequest(digest=digest,
args=(os.path.join("{chroot}", "program.py"), ))
async def docker_image_run_request(
field_set: DockerFieldSet, docker: DockerBinary, options: DockerOptions
) -> RunRequest:
env, image = await MultiGet(
Get(Environment, EnvironmentRequest(options.env_vars)),
Get(BuiltPackage, PackageFieldSet, field_set),
)
tag = cast(BuiltDockerImage, image.artifacts[0]).tags[0]
run = docker.run_image(tag, docker_run_args=options.run_args, env=env)
return RunRequest(args=run.argv, digest=image.digest, extra_env=run.env)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources,
HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()))
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
output_filename = f"{field_set.address.target_name}.pex"
pex_request = Get(
Pex,
PexFromTargetsRequest(
addresses=Addresses([field_set.address]),
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
include_source_files=False,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True),
)
pex, sources = await MultiGet(pex_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests([pex.digest, sources.source_files.snapshot.digest]))
return RunRequest(
digest=merged_digest,
binary_name=pex.output_filename,
prefix_args=("-m", entry_point),
env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
)
def create_mock_run_request(rule_runner: RuleRunner,
program_text: bytes) -> RunRequest:
digest = rule_runner.request(
Digest,
[
CreateDigest([
FileContent(path="program.py",
content=program_text,
is_executable=True)
])
],
)
return RunRequest(digest=digest,
args=(os.path.join("{chroot}", "program.py"), ))
async def docker_image_run_request(field_set: DockerFieldSet,
docker: DockerBinary) -> RunRequest:
image = await Get(BuiltPackage, PackageFieldSet, field_set)
return RunRequest(
digest=image.digest,
args=tuple(
cast(str, arg) for arg in (
docker.path,
"run",
"-it" if sys.stdout.isatty() else False,
"--rm",
cast(BuiltDockerImage, image.artifacts[0]).tags[0],
) if arg),
)
async def run_shell_command_request(
shell_command: RunShellCommand) -> RunRequest:
wrapped_tgt = await Get(
WrappedTarget,
WrappedTargetRequest(shell_command.address,
description_of_origin="<infallible>"),
)
process = await Get(Process,
ShellCommandProcessRequest(wrapped_tgt.target))
return RunRequest(
digest=process.input_digest,
args=process.argv,
extra_env=process.env,
)
async def run_pyoxidizer_binary(field_set: PyOxidizerFieldSet) -> RunRequest:
def is_executable_binary(artifact_relpath: str | None) -> bool:
"""After packaging, the PyOxidizer plugin will place the executable in a location like this:
dist/{project}/{target_name}/{platform arch}/{compilation mode}/install/{binary name}
{binary name} will default to `target_name`, but can be modified with a custom PyOxidizer template.
e.g. dist/helloworld/helloworld-bin/x86_64-apple-darwin/debug/install/helloworld-bin.
PyOxidizer will place associated libraries in {...}/install/lib
To determine if the artifact we iterate over is the one we want to execute, we check that
the file's parent dir is "install". There should only be one of these files.
"""
if not artifact_relpath:
return False
artifact_path = PurePath(artifact_relpath)
return artifact_path.parent.name == "install"
binary = await Get(BuiltPackage, PackageFieldSet, field_set)
executable_binaries = [
artifact for artifact in binary.artifacts
if is_executable_binary(artifact.relpath)
]
assert len(executable_binaries) == 1, softwrap(f"""
More than one executable binary discovered in the `install` directory,
which is a bug in the PyOxidizer plugin.
Please file a bug report at https://github.com/pantsbuild/pants/issues/new.
Enumerated executable binaries: {executable_binaries}
""")
artifact = executable_binaries[0]
assert artifact.relpath is not None
return RunRequest(digest=binary.digest,
args=(os.path.join("{chroot}", artifact.relpath), ))
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment) -> RunRequest:
run_in_sandbox = field_set.run_in_sandbox.value
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
addresses = [field_set.address]
interpreter_constraints = await Get(
InterpreterConstraints, InterpreterConstraintsRequest(addresses))
pex_filename = (field_set.address.generated_name.replace(".", "_")
if field_set.address.generated_name else
field_set.address.target_name)
pex_get = Get(
Pex,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{pex_filename}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
input_digests = [
pex.digest,
local_dists.pex.digest,
# Note regarding inline mode: You might think that the sources don't need to be copied
# into the chroot when using inline sources. But they do, because some of them might be
# codegenned, and those won't exist in the inline source tree. Rather than incurring the
# complexity of figuring out here which sources were codegenned, we copy everything.
# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
# sources will take precedence and their copies in the chroot will be ignored.
local_dists.remaining_sources.source_files.snapshot.digest,
]
merged_digest = await Get(Digest, MergeDigests(input_digests))
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
# The order here is important: we want the in-repo sources to take precedence over their
# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
source_roots = [
*([] if run_in_sandbox else sources.source_roots),
*chrooted_source_roots,
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the PEX itself. It's loaded by setting
# `PEX_EXTRA_SYS_PATH`.
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=entry_point.val,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(Addresses([field_set.address]), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)
)
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.interpreter_constraints,
additional_args=field_set.generate_additional_args(python_binary_defaults),
internal_only=True,
),
)
requirements, sources, runner_pex = await MultiGet(
requirements_request, sources_request, runner_pex_request
)
merged_digest = await Get(
Digest,
MergeDigests(
[requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest]
),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict,
"PEX_PATH": in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
}
return RunRequest(
digest=merged_digest,
args=(in_chroot(runner_pex.name), "-m", entry_point),
extra_env=extra_env,
)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the Pex itself, but on the
# PEX_PATH. This works fine!
entry_point=entry_point,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
python_setup: PythonSetup,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in local_dists_get.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{field_set.address.target_name}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
python_setup),
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
pex_get = Get(Pex, PexRequest, requirements_pex_request)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests([
pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_deploy_jar_run_request(
field_set: DeployJarFieldSet, ) -> RunRequest:
jdk = await Get(JdkEnvironment, JdkRequest,
JdkRequest.from_field(field_set.jdk_version))
main_class = field_set.main_class.value
assert main_class is not None
package = await Get(BuiltPackage, DeployJarFieldSet, field_set)
assert len(package.artifacts) == 1
jar_path = package.artifacts[0].relpath
assert jar_path is not None
proc = await Get(
Process,
JvmProcess(
jdk=jdk,
classpath_entries=[f"{{chroot}}/{jar_path}"],
argv=(main_class, ),
input_digest=package.digest,
description=f"Run {main_class}.main(String[])",
use_nailgun=False,
),
)
support_digests = await MultiGet(
Get(Digest, AddPrefix(digest, prefix))
for prefix, digest in proc.immutable_input_digests.items())
runtime_jvm = await Get(__RuntimeJvm, JdkEnvironment, jdk)
support_digests += (runtime_jvm.digest, )
# TODO(#14386) This argument re-writing code should be done in a more standardised way.
# See also `jdk_rules.py` for other argument re-writing code.
def prefixed(arg: str, prefixes: Iterable[str]) -> str:
if any(arg.startswith(prefix) for prefix in prefixes):
return f"{{chroot}}/{arg}"
else:
return arg
prefixes = (jdk.bin_dir, jdk.jdk_preparation_script, jdk.java_home)
args = [prefixed(arg, prefixes) for arg in proc.argv]
env = {
**proc.env,
"PANTS_INTERNAL_ABSOLUTE_PREFIX": "{chroot}/",
}
# absolutify coursier cache envvars
for key in env:
if key.startswith("COURSIER"):
env[key] = prefixed(env[key], (jdk.coursier.cache_dir, ))
request_digest = await Get(
Digest,
MergeDigests([
proc.input_digest,
*support_digests,
]),
)
return RunRequest(
digest=request_digest,
args=args,
extra_env=env,
)