async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
resolved_entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point))
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
def test_issue_12222(rule_runner: RuleRunner) -> None:
rule_runner.write_files({
"constraints.txt":
"foo==1.0\nbar==1.0",
"BUILD":
dedent("""
python_requirement(name="foo",requirements=["foo"])
python_requirement(name="bar",requirements=["bar"])
python_sources(name="lib",sources=[],dependencies=[":foo"])
"""),
})
request = PexFromTargetsRequest(
[Address("", target_name="lib")],
output_filename="demo.pex",
internal_only=False,
platforms=PexPlatforms(["some-platform-x86_64"]),
)
rule_runner.set_options([
"--python-requirement-constraints=constraints.txt",
"--python-resolve-all-constraints",
])
result = rule_runner.request(PexRequest, [request])
assert result.requirements == PexRequirements(["foo"],
apply_constraints=True)
def create_pex_and_get_pex_info(
rule_runner: RuleRunner,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
platforms=PexPlatforms(),
sources: Optional[Digest] = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
internal_only: bool = True,
) -> Dict:
return cast(
Dict,
create_pex_and_get_all_data(
rule_runner,
requirements=requirements,
entry_point=entry_point,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
sources=sources,
additional_pants_args=additional_pants_args,
additional_pex_args=additional_pex_args,
internal_only=internal_only,
)["info"],
)
def create_pex_and_get_pex_info(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints:
PexInterpreterConstraints = PexInterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
internal_only: bool = True,
) -> Dict:
return cast(
Dict,
create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
requirements=requirements,
main=main,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
sources=sources,
additional_pants_args=additional_pants_args,
additional_pex_args=additional_pex_args,
internal_only=internal_only,
)["info"],
)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
softwrap(f"""
The `pex_binary` target {field_set.address} transitively depends on the below `files`
targets, but Pants will not include them in the PEX. Filesystem APIs like `open()`
are not able to load files within the binary itself; instead, they read from the
current working directory.
Instead, use `resources` targets or wrap this `pex_binary` in an `archive`.
See {doc_url('resources')}.
Files targets dependencies: {files_addresses}
"""))
output_filename = field_set.output_path.value_or_default(file_ending="pex")
complete_platforms = await Get(CompletePlatforms,
PexCompletePlatformsField,
field_set.complete_platforms)
pex = await Get(
Pex,
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=resolved_entry_point.val or field_set.script.value,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
complete_platforms=complete_platforms,
output_filename=output_filename,
layout=PexLayout(field_set.layout.value),
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
include_requirements=field_set.include_requirements.value,
include_local_dists=True,
),
)
return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints:
PexInterpreterConstraints = PexInterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_inputs: Digest | None = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
env: Mapping[str, str] | None = None,
internal_only: bool = True,
) -> Dict:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
main=main,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env,
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
pex = rule_runner.request(pex_type, [request])
if isinstance(pex, Pex):
digest = pex.digest
elif isinstance(pex, VenvPex):
digest = pex.digest
else:
raise AssertionError(
f"Expected a Pex or a VenvPex but got a {type(pex)}.")
rule_runner.scheduler.write_digest(digest)
pex_path = os.path.join(rule_runner.build_root, "test.pex")
with zipfile.ZipFile(pex_path, "r") as zipfp:
with zipfp.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
pex_list = zipfp.namelist()
return {
"pex": pex,
"local_path": pex_path,
"info": json.loads(pex_info_content),
"files": pex_list,
}
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
files_tgts = targets_with_sources_types([FilesSources],
transitive_targets.dependencies,
union_membership)
if files_tgts:
files_addresses = sorted(tgt.address.spec for tgt in files_tgts)
logger.warning(
f"The pex_binary target {field_set.address} transitively depends on the below files "
"targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` "
"are not able to load files within the binary itself; instead, they read from the "
"current working directory."
"\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See "
f"{bracketed_docs_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
def __init__(
self,
addresses: Iterable[Address],
*,
internal_only: bool,
hardcoded_interpreter_constraints: InterpreterConstraints
| None = None,
platforms: PexPlatforms = PexPlatforms(),
complete_platforms: CompletePlatforms = CompletePlatforms(),
additional_lockfile_args: tuple[str, ...] = (),
) -> None:
self.addresses = Addresses(addresses)
self.internal_only = internal_only
self.hardcoded_interpreter_constraints = hardcoded_interpreter_constraints
self.platforms = platforms
self.complete_platforms = complete_platforms
self.additional_lockfile_args = additional_lockfile_args
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
global_options: GlobalOptions,
) -> BuiltPackage:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PexBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
output_filename = field_set.output_path.value_or_default(
field_set.address,
file_ending="pex",
use_legacy_format=global_options.options.pants_distdir_legacy_paths,
)
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
def test_platforms(rule_runner: RuleRunner) -> None:
# We use Python 2.7, rather than Python 3, to ensure that the specified platform is
# actually used.
platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"])
constraints = PexInterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"])
pex_output = create_pex_and_get_all_data(
rule_runner,
requirements=PexRequirements(["cryptography==2.9"]),
platforms=platforms,
interpreter_constraints=constraints,
internal_only=False, # Internal only PEXes do not support (foreign) platforms.
)
assert any(
"cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp for fp in pex_output["files"]
)
assert not any("cryptography-2.9-cp27-cp27m-" in fp for fp in pex_output["files"])
assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_output["files"])
# NB: Platforms override interpreter constraints.
assert pex_output["info"]["interpreter_constraints"] == []
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
platforms=PexPlatforms(),
sources: Optional[Digest] = None,
additional_inputs: Optional[Digest] = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
env: Optional[Mapping[str, str]] = None,
internal_only: bool = True,
) -> Dict:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
entry_point=entry_point,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env)
pex = rule_runner.request(Pex, [request])
rule_runner.scheduler.write_digest(pex.digest)
pex_path = os.path.join(rule_runner.build_root, "test.pex")
with zipfile.ZipFile(pex_path, "r") as zipfp:
with zipfp.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
pex_list = zipfp.namelist()
return {
"pex": pex,
"local_path": pex_path,
"info": json.loads(pex_info_content),
"files": pex_list,
}
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements | EntireLockfile = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints: InterpreterConstraints = InterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_inputs: Digest | None = None,
additional_pants_args: tuple[str, ...] = (),
additional_pex_args: tuple[str, ...] = (),
env: Mapping[str, str] | None = None,
internal_only: bool = True,
layout: PexLayout | None = None,
) -> PexData:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
main=main,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
layout=layout,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env,
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
pex: Pex | VenvPex
if pex_type == Pex:
pex = rule_runner.request(Pex, [request])
else:
pex = rule_runner.request(VenvPex, [request])
return get_all_data(rule_runner, pex)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
resolved_entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point))
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex
) -> BuiltPackage:
output_filename = field_set.output_path.value_or_default(
field_set.address,
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip",
)
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=None,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=[
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
],
)
)
lambdex_request = PexRequest(
output_filename="lambdex.pex",
internal_only=True,
requirements=PexRequirements(lambdex.all_requirements),
interpreter_constraints=PexInterpreterConstraints(lambdex.interpreter_constraints),
main=lambdex.main,
)
lambdex_pex, pex_result, handler = await MultiGet(
Get(VenvPex, PexRequest, lambdex_request),
Get(TwoStepPex, TwoStepPexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)),
)
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.pex.digest,
output_files=(output_filename,),
description=f"Setting up handler in {output_filename}",
),
)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=(
f" Runtime: {field_set.runtime.value}",
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
" Handler: lambdex_handler.handler",
),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact,))
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints: PexInterpreterConstraints = PexInterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_inputs: Digest | None = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
env: Mapping[str, str] | None = None,
internal_only: bool = True,
) -> Dict:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
main=main,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env,
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
pex = rule_runner.request(pex_type, [request])
if isinstance(pex, Pex):
digest = pex.digest
pex_pex = rule_runner.request(PexPEX, [])
process = rule_runner.request(
Process,
[
PexProcess(
Pex(digest=pex_pex.digest, name=pex_pex.exe, python=pex.python),
argv=["-m", "pex.tools", pex.name, "info"],
input_digest=pex.digest,
extra_env=dict(PEX_INTERPRETER="1"),
description="Extract PEX-INFO.",
)
],
)
elif isinstance(pex, VenvPex):
digest = pex.digest
process = rule_runner.request(
Process,
[
VenvPexProcess(
pex,
argv=["info"],
extra_env=dict(PEX_TOOLS="1"),
description="Extract PEX-INFO.",
),
],
)
else:
raise AssertionError(f"Expected a Pex or a VenvPex but got a {type(pex)}.")
rule_runner.scheduler.write_digest(digest)
pex_path = os.path.join(rule_runner.build_root, "test.pex")
result = rule_runner.request(ProcessResult, [process])
pex_info_content = result.stdout.decode()
with zipfile.ZipFile(pex_path, "r") as zipfp:
pex_list = zipfp.namelist()
return {
"pex": pex,
"local_path": pex_path,
"info": json.loads(pex_info_content),
"files": pex_list,
}
def __init__(
self,
addresses: Iterable[Address],
*,
output_filename: str,
internal_only: bool,
main: MainSpecification | None = None,
platforms: PexPlatforms = PexPlatforms(),
additional_args: Iterable[str] = (),
additional_requirements: Iterable[str] = (),
include_source_files: bool = True,
additional_sources: Digest | None = None,
additional_inputs: Digest | None = None,
hardcoded_interpreter_constraints: PexInterpreterConstraints | None = None,
direct_deps_only: bool = False,
description: str | None = None,
) -> None:
"""Request to create a Pex from the transitive closure of the given addresses.
:param addresses: The addresses to use for determining what is included in the Pex. The
transitive closure of these addresses will be used; you only need to specify the roots.
:param output_filename: The name of the built Pex file, which typically should end in
`.pex`.
:param internal_only: Whether we ever materialize the Pex and distribute it directly
to end users, such as with the `binary` goal. Typically, instead, the user never
directly uses the Pex, e.g. with `lint` and `test`. If True, we will use a Pex setting
that results in faster build time but compatibility with fewer interpreters at runtime.
:param main: The main for the built Pex, equivalent to Pex's `-e` or `-c` flag. If
left off, the Pex will open up as a REPL.
:param platforms: Which platforms should be supported. Setting this value will cause
interpreter constraints to not be used because platforms already constrain the valid
Python versions, e.g. by including `cp36m` in the platform string.
:param additional_args: Any additional Pex flags.
:param additional_requirements: Additional requirements to install, in addition to any
requirements used by the transitive closure of the given addresses.
:param include_source_files: Whether to include source files in the built Pex or not.
Setting this to `False` and loading the source files by instead populating the chroot
and setting the environment variable `PEX_EXTRA_SYS_PATH` will result in substantially
fewer rebuilds of the Pex.
:param additional_sources: Any additional source files to include in the built Pex.
:param additional_inputs: Any inputs that are not source files and should not be included
directly in the Pex, but should be present in the environment when building the Pex.
:param hardcoded_interpreter_constraints: Use these constraints rather than resolving the
constraints from the input.
:param direct_deps_only: Only consider the input addresses and their direct dependencies,
rather than the transitive closure.
:param description: A human-readable description to render in the dynamic UI when building
the Pex.
"""
self.addresses = Addresses(addresses)
self.output_filename = output_filename
self.internal_only = internal_only
self.main = main
self.platforms = platforms
self.additional_args = tuple(additional_args)
self.additional_requirements = tuple(additional_requirements)
self.include_source_files = include_source_files
self.additional_sources = additional_sources
self.additional_inputs = additional_inputs
self.hardcoded_interpreter_constraints = hardcoded_interpreter_constraints
self.direct_deps_only = direct_deps_only
self.description = description
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements | Lockfile
| LockfileContent = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints: InterpreterConstraints = InterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_inputs: Digest | None = None,
additional_pants_args: tuple[str, ...] = (),
additional_pex_args: tuple[str, ...] = (),
env: Mapping[str, str] | None = None,
internal_only: bool = True,
) -> PexData:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
main=main,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env,
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
pex: Pex | VenvPex
if pex_type == Pex:
pex = rule_runner.request(Pex, [request])
digest = pex.digest
sandbox_path = pex.name
pex_pex = rule_runner.request(PexPEX, [])
process = rule_runner.request(
Process,
[
PexProcess(
Pex(digest=pex_pex.digest,
name=pex_pex.exe,
python=pex.python),
argv=["-m", "pex.tools", pex.name, "info"],
input_digest=pex.digest,
extra_env=dict(PEX_INTERPRETER="1"),
description="Extract PEX-INFO.",
)
],
)
else:
pex = rule_runner.request(VenvPex, [request])
digest = pex.digest
sandbox_path = pex.pex_filename
process = rule_runner.request(
Process,
[
VenvPexProcess(
pex,
argv=["info"],
extra_env=dict(PEX_TOOLS="1"),
description="Extract PEX-INFO.",
),
],
)
rule_runner.scheduler.write_digest(digest)
local_path = PurePath(rule_runner.build_root) / "test.pex"
result = rule_runner.request(ProcessResult, [process])
pex_info_content = result.stdout.decode()
is_zipapp = zipfile.is_zipfile(local_path)
if is_zipapp:
with zipfile.ZipFile(local_path, "r") as zipfp:
files = tuple(zipfp.namelist())
else:
files = tuple(
os.path.normpath(
os.path.relpath(os.path.join(root, path), local_path))
for root, dirs, files in os.walk(local_path)
for path in dirs + files)
return PexData(
pex=pex,
is_zipapp=is_zipapp,
sandbox_path=PurePath(sandbox_path),
local_path=local_path,
info=json.loads(pex_info_content),
files=files,
)
async def create_python_awslambda(
field_set: PythonAwsLambdaFieldSet, lambdex_setup: LambdexSetup, global_options: GlobalOptions
) -> CreatedAWSLambda:
output_filename = field_set.output_path.value_or_default(
field_set.address,
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip",
use_legacy_format=global_options.options.pants_distdir_legacy_paths,
)
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=None,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=[
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
],
)
)
pex_result = await Get(TwoStepPex, TwoStepPexFromTargetsRequest, pex_request)
input_digest = await Get(
Digest, MergeDigests((pex_result.pex.digest, lambdex_setup.requirements_pex.digest))
)
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
PexProcess(
lambdex_setup.requirements_pex,
argv=("build", "-e", field_set.handler.value, output_filename),
input_digest=input_digest,
output_files=(output_filename,),
description=f"Setting up handler in {output_filename}",
),
)
return CreatedAWSLambda(
digest=result.output_digest,
zip_file_relpath=output_filename,
runtime=field_set.runtime.value,
# The AWS-facing handler function is always lambdex_handler.handler, which is the wrapper
# injected by lambdex that manages invocation of the actual handler.
handler="lambdex_handler.handler",
)
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet,
lambdex: Lambdex,
platform: Platform,
union_membership: UnionMembership,
) -> BuiltPackage:
if platform.is_macos:
logger.warning(
"AWS Lambdas built on macOS may fail to build. If your lambda uses any third-party"
" dependencies without binary wheels (bdist) for Linux available, it will fail to"
" build. If this happens, you will either need to update your dependencies to only use"
f" dependencies with pre-built wheels, or find a Linux environment to run {bin_name()}"
" package. (See https://realpython.com/python-wheels/ for more about wheels.)\n\n(If"
" the build does not raise an exception, it's safe to use macOS.)")
output_filename = field_set.output_path.value_or_default(
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip", )
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
pex_platforms = []
interpreter_version = field_set.runtime.to_interpreter_version()
if interpreter_version:
py_major, py_minor = interpreter_version
platform_str = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform_str += "m"
if (py_major, py_minor) == (2, 7):
platform_str += "u"
pex_platforms.append(platform_str)
additional_pex_args = (
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
)
complete_platforms = await Get(CompletePlatforms,
PexCompletePlatformsField,
field_set.complete_platforms)
pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
include_requirements=field_set.include_requirements.value,
output_filename=output_filename,
platforms=PexPlatforms(pex_platforms),
complete_platforms=complete_platforms,
additional_args=additional_pex_args,
additional_lockfile_args=additional_pex_args,
)
lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex.to_pex_request()),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler,
ResolvePythonAwsHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
softwrap(f"""
The `python_awslambda` target {field_set.address} transitively depends on the below
`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs
like `open()` are not able to load files within the binary itself; instead, they
read from the current working directory.
Instead, use `resources` targets. See {doc_url('resources')}.
Files targets dependencies: {files_addresses}
"""))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.digest,
output_files=(output_filename, ),
description=f"Setting up handler in {output_filename}",
),
)
extra_log_data: list[tuple[str, str]] = []
if field_set.runtime.value:
extra_log_data.append(("Runtime", field_set.runtime.value))
extra_log_data.extend(
("Complete platform", path) for path in complete_platforms)
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
extra_log_data.append(("Handler", "lambdex_handler.handler"))
first_column_width = 4 + max(len(header) for header, _ in extra_log_data)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=tuple(
f"{header.rjust(first_column_width, ' ')}: {data}"
for header, data in extra_log_data),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex,
union_membership: UnionMembership) -> BuiltPackage:
output_filename = field_set.output_path.value_or_default(
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip", )
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
additional_pex_args = (
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
)
pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=additional_pex_args,
additional_lockfile_args=additional_pex_args,
)
lambdex_request = PexRequest(
output_filename="lambdex.pex",
internal_only=True,
requirements=lambdex.pex_requirements(),
interpreter_constraints=lambdex.interpreter_constraints,
main=lambdex.main,
)
lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex_request),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler,
ResolvePythonAwsHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
f"The `python_awslambda` target {field_set.address} transitively depends on the below "
"`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs "
"like `open()` are not able to load files within the binary itself; instead, they "
"read from the current working directory."
f"\n\nInstead, use `resources` targets. See {doc_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.digest,
output_files=(output_filename, ),
description=f"Setting up handler in {output_filename}",
),
)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=(
f" Runtime: {field_set.runtime.value}",
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
" Handler: lambdex_handler.handler",
),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
