async def create_python_binary(
field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults
) -> CreatedBinary:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
output_filename = f"{field_set.address.target_name}.pex"
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(python_binary_defaults),
)
),
)
pex = two_step_pex.pex
return CreatedBinary(digest=pex.digest, binary_name=pex.output_filename)
async def create_python_binary(
config: PythonBinaryConfiguration) -> CreatedBinary:
entry_point: Optional[str]
if config.entry_point.value is not None:
entry_point = config.entry_point.value
else:
source_files = await Get[SourceFiles](AllSourceFilesRequest(
[config.sources], strip_source_roots=True))
# NB: `PythonBinarySources` enforces that we have 0-1 sources.
if len(source_files.files) == 1:
module_name = source_files.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
module_name)
else:
entry_point = None
output_filename = f"{config.address.target_name}.pex"
two_step_pex = await Get[TwoStepPex](TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=Addresses([config.address]),
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(
config.platforms),
output_filename=output_filename,
additional_args=config.generate_additional_args(),
)))
pex = two_step_pex.pex
return CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def create_python_awslambda(
field_set: PythonAwsLambdaFieldSet,
lambdex_setup: LambdexSetup) -> CreatedAWSLambda:
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
pex_filename = f"{field_set.address.target_name}.zip"
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=None,
output_filename=pex_filename,
platforms=PexPlatforms([platform]),
additional_args=[
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
],
))
pex_result = await Get(TwoStepPex, TwoStepPexFromTargetsRequest,
pex_request)
input_digest = await Get(
Digest,
MergeDigests(
(pex_result.pex.digest, lambdex_setup.requirements_pex.digest)))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
PexProcess(
lambdex_setup.requirements_pex,
argv=("build", "-e", field_set.handler.value, pex_filename),
input_digest=input_digest,
output_files=(pex_filename, ),
description=f"Setting up handler in {pex_filename}",
),
)
return CreatedAWSLambda(
digest=result.output_digest,
zip_file_relpath=pex_filename,
runtime=field_set.runtime.value,
# The AWS-facing handler function is always lambdex_handler.handler, which is the wrapper
# injected by lambdex that manages invocation of the actual handler.
handler="lambdex_handler.handler",
)
def __init__(
self,
addresses: Iterable[Address],
*,
output_filename: str,
internal_only: bool,
entry_point: Optional[str] = None,
platforms: PexPlatforms = PexPlatforms(),
additional_args: Iterable[str] = (),
additional_requirements: Iterable[str] = (),
include_source_files: bool = True,
additional_sources: Optional[Digest] = None,
additional_inputs: Optional[Digest] = None,
description: Optional[str] = None,
) -> None:
"""Request to create a Pex from the transitive closure of the given addresses.
:param addresses: The addresses to use for determining what is included in the Pex. The
transitive closure of these addresses will be used; you only need to specify the roots.
:param output_filename: The name of the built Pex file, which typically should end in
`.pex`.
:param internal_only: Whether we ever materialize the Pex and distribute it directly
to end users, such as with the `binary` goal. Typically, instead, the user never
directly uses the Pex, e.g. with `lint` and `test`. If True, we will use a Pex setting
that results in faster build time but compatibility with fewer interpreters at runtime.
:param entry_point: The entry-point for the built Pex, equivalent to Pex's `-m` flag. If
left off, the Pex will open up as a REPL.
:param platforms: Which platforms should be supported. Setting this value will cause
interpreter constraints to not be used because platforms already constrain the valid
Python versions, e.g. by including `cp36m` in the platform string.
:param additional_args: Any additional Pex flags.
:param additional_requirements: Additional requirements to install, in addition to any
requirements used by the transitive closure of the given addresses.
:param include_source_files: Whether to include source files in the built Pex or not.
Setting this to `False` and loading the source files by instead populating the chroot
and setting the environment variable `PEX_EXTRA_SYS_PATH` will result in substantially
fewer rebuilds of the Pex.
:param additional_sources: Any additional source files to include in the built Pex.
:param additional_inputs: Any inputs that are not source files and should not be included
directly in the Pex, but should be present in the environment when building the Pex.
:param description: A human-readable description to render in the dynamic UI when building
the Pex.
"""
self.addresses = Addresses(addresses)
self.output_filename = output_filename
self.internal_only = internal_only
self.entry_point = entry_point
self.platforms = platforms
self.additional_args = tuple(additional_args)
self.additional_requirements = tuple(additional_requirements)
self.include_source_files = include_source_files
self.additional_sources = additional_sources
self.additional_inputs = additional_inputs
self.description = description
async def create_python_awslambda(
field_set: PythonAwsLambdaFieldSet,
lambdex_setup: LambdexSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CreatedAWSLambda:
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
pex_filename = f"{field_set.address.target_name}.zip"
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"manylinux2014_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=Addresses([field_set.address]),
entry_point=None,
output_filename=pex_filename,
platforms=PexPlatforms([platform]),
))
pex_result = await Get[TwoStepPex](TwoStepPexFromTargetsRequest,
pex_request)
input_digest = await Get[Digest](MergeDigests(
(pex_result.pex.digest, lambdex_setup.requirements_pex.digest)))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
lambdex_args = ("build", "-e", field_set.handler.value, pex_filename)
process = lambdex_setup.requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./lambdex.pex",
pex_args=lambdex_args,
input_digest=input_digest,
output_files=(pex_filename, ),
description=f"Setting up handler in {pex_filename}",
)
result = await Get[ProcessResult](Process, process)
# Note that the AWS-facing handler function is always lambdex_handler.handler, which
# is the wrapper injected by lambdex that manages invocation of the actual handler.
return CreatedAWSLambda(
digest=result.output_digest,
name=pex_filename,
runtime=field_set.runtime.value,
handler="lambdex_handler.handler",
)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources,
HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()))
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
output_filename = f"{field_set.address.target_name}.pex"
pex_request = Get(
Pex,
PexFromTargetsRequest(
addresses=Addresses([field_set.address]),
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
include_source_files=False,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True),
)
pex, sources = await MultiGet(pex_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests([pex.digest, sources.source_files.snapshot.digest]))
return RunRequest(
digest=merged_digest,
binary_name=pex.output_filename,
prefix_args=("-m", entry_point),
env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
)
def create_pex_and_get_all_data(
self,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
platforms=PexPlatforms(),
sources: Optional[Digest] = None,
additional_inputs: Optional[Digest] = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
) -> Dict:
request = PexRequest(
output_filename="test.pex",
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
entry_point=entry_point,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
pex = self.request_single_product(
Pex,
Params(
request,
create_options_bootstrapper(args=[
"--backend-packages2=pants.backend.python",
*additional_pants_args
]),
),
)
self.scheduler.materialize_directory(DirectoryToMaterialize(
pex.digest))
pex_path = os.path.join(self.build_root, "test.pex")
with zipfile.ZipFile(pex_path, "r") as zipfp:
with zipfp.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
pex_list = zipfp.namelist()
return {
"pex": pex,
"local_path": pex_path,
"info": json.loads(pex_info_content),
"files": pex_list,
}
def test_platforms(self) -> None:
# We use Python 2.7, rather than Python 3, to ensure that the specified platform is
# actually used.
platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"])
constraints = PexInterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"])
pex_output = self.create_pex_and_get_all_data(
requirements=PexRequirements(["cryptography==2.9"]),
platforms=platforms,
interpreter_constraints=constraints,
)
assert any(
"cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp
for fp in pex_output["files"]
)
assert not any("cryptography-2.9-cp27-cp27m-" in fp for fp in pex_output["files"])
assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_output["files"])
# NB: Platforms override interpreter constraints.
assert pex_output["info"]["interpreter_constraints"] == []
async def create_python_binary(
field_set: PythonBinaryFieldSet) -> CreatedBinary:
entry_point = field_set.entry_point.value
if entry_point is None:
source_files = await Get[SourceFiles](AllSourceFilesRequest(
[field_set.sources], strip_source_roots=True))
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
source_files.files)
output_filename = f"{field_set.address.target_name}.pex"
two_step_pex = await Get[TwoStepPex](TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=Addresses([field_set.address]),
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(),
)))
pex = two_step_pex.pex
return CreatedBinary(digest=pex.digest, binary_name=pex.output_filename)
def __init__(self,
addresses: Addresses,
*,
output_filename: str,
entry_point: Optional[str] = None,
platforms: PexPlatforms = PexPlatforms(),
additional_args: Iterable[str] = (),
additional_requirements: Iterable[str] = (),
include_source_files: bool = True,
additional_sources: Optional[Digest] = None,
additional_inputs: Optional[Digest] = None,
description: Optional[str] = None) -> None:
self.addresses = addresses
self.output_filename = output_filename
self.entry_point = entry_point
self.platforms = platforms
self.additional_args = tuple(additional_args)
self.additional_requirements = tuple(additional_requirements)
self.include_source_files = include_source_files
self.additional_sources = additional_sources
self.additional_inputs = additional_inputs
self.description = description
def create_pex_and_get_pex_info(
self,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
platforms=PexPlatforms(),
sources: Optional[Digest] = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
) -> Dict:
return cast(
Dict,
self.create_pex_and_get_all_data(
requirements=requirements,
entry_point=entry_point,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
sources=sources,
additional_pants_args=additional_pants_args,
additional_pex_args=additional_pex_args,
)["info"],
)
