async def create_python_binary( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults ) -> CreatedBinary: entry_point = field_set.entry_point.value if entry_point is None: # TODO: This is overkill? We don't need to hydrate the sources and strip snapshots, # we only need the path relative to the source root. binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources)) stripped_binary_sources = await Get( StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()) ) entry_point = PythonBinarySources.translate_source_file_to_entry_point( stripped_binary_sources.snapshot.files ) output_filename = f"{field_set.address.target_name}.pex" two_step_pex = await Get( TwoStepPex, TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], entry_point=entry_point, platforms=PexPlatforms.create_from_platforms_field(field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args(python_binary_defaults), ) ), ) pex = two_step_pex.pex return CreatedBinary(digest=pex.digest, binary_name=pex.output_filename)
async def create_python_binary( config: PythonBinaryConfiguration) -> CreatedBinary: entry_point: Optional[str] if config.entry_point.value is not None: entry_point = config.entry_point.value else: source_files = await Get[SourceFiles](AllSourceFilesRequest( [config.sources], strip_source_roots=True)) # NB: `PythonBinarySources` enforces that we have 0-1 sources. if len(source_files.files) == 1: module_name = source_files.files[0] entry_point = PythonBinary.translate_source_path_to_py_module_specifier( module_name) else: entry_point = None output_filename = f"{config.address.target_name}.pex" two_step_pex = await Get[TwoStepPex](TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=Addresses([config.address]), entry_point=entry_point, platforms=PexPlatforms.create_from_platforms_field( config.platforms), output_filename=output_filename, additional_args=config.generate_additional_args(), ))) pex = two_step_pex.pex return CreatedBinary(digest=pex.directory_digest, binary_name=pex.output_filename)
async def create_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex_setup: LambdexSetup) -> CreatedAWSLambda: # Lambdas typically use the .zip suffix, so we use that instead of .pex. pex_filename = f"{field_set.address.target_name}.zip" # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) py_major, py_minor = field_set.runtime.to_interpreter_version() platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform += "m" if (py_major, py_minor) == (2, 7): platform += "u" pex_request = TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, entry_point=None, output_filename=pex_filename, platforms=PexPlatforms([platform]), additional_args=[ # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ], )) pex_result = await Get(TwoStepPex, TwoStepPexFromTargetsRequest, pex_request) input_digest = await Get( Digest, MergeDigests( (pex_result.pex.digest, lambdex_setup.requirements_pex.digest))) # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, PexProcess( lambdex_setup.requirements_pex, argv=("build", "-e", field_set.handler.value, pex_filename), input_digest=input_digest, output_files=(pex_filename, ), description=f"Setting up handler in {pex_filename}", ), ) return CreatedAWSLambda( digest=result.output_digest, zip_file_relpath=pex_filename, runtime=field_set.runtime.value, # The AWS-facing handler function is always lambdex_handler.handler, which is the wrapper # injected by lambdex that manages invocation of the actual handler. handler="lambdex_handler.handler", )
def __init__( self, addresses: Iterable[Address], *, output_filename: str, internal_only: bool, entry_point: Optional[str] = None, platforms: PexPlatforms = PexPlatforms(), additional_args: Iterable[str] = (), additional_requirements: Iterable[str] = (), include_source_files: bool = True, additional_sources: Optional[Digest] = None, additional_inputs: Optional[Digest] = None, description: Optional[str] = None, ) -> None: """Request to create a Pex from the transitive closure of the given addresses. :param addresses: The addresses to use for determining what is included in the Pex. The transitive closure of these addresses will be used; you only need to specify the roots. :param output_filename: The name of the built Pex file, which typically should end in `.pex`. :param internal_only: Whether we ever materialize the Pex and distribute it directly to end users, such as with the `binary` goal. Typically, instead, the user never directly uses the Pex, e.g. with `lint` and `test`. If True, we will use a Pex setting that results in faster build time but compatibility with fewer interpreters at runtime. :param entry_point: The entry-point for the built Pex, equivalent to Pex's `-m` flag. If left off, the Pex will open up as a REPL. :param platforms: Which platforms should be supported. Setting this value will cause interpreter constraints to not be used because platforms already constrain the valid Python versions, e.g. by including `cp36m` in the platform string. :param additional_args: Any additional Pex flags. :param additional_requirements: Additional requirements to install, in addition to any requirements used by the transitive closure of the given addresses. :param include_source_files: Whether to include source files in the built Pex or not. Setting this to `False` and loading the source files by instead populating the chroot and setting the environment variable `PEX_EXTRA_SYS_PATH` will result in substantially fewer rebuilds of the Pex. :param additional_sources: Any additional source files to include in the built Pex. :param additional_inputs: Any inputs that are not source files and should not be included directly in the Pex, but should be present in the environment when building the Pex. :param description: A human-readable description to render in the dynamic UI when building the Pex. """ self.addresses = Addresses(addresses) self.output_filename = output_filename self.internal_only = internal_only self.entry_point = entry_point self.platforms = platforms self.additional_args = tuple(additional_args) self.additional_requirements = tuple(additional_requirements) self.include_source_files = include_source_files self.additional_sources = additional_sources self.additional_inputs = additional_inputs self.description = description
async def create_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex_setup: LambdexSetup, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> CreatedAWSLambda: # Lambdas typically use the .zip suffix, so we use that instead of .pex. pex_filename = f"{field_set.address.target_name}.zip" # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) py_major, py_minor = field_set.runtime.to_interpreter_version() platform = f"manylinux2014_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform += "m" if (py_major, py_minor) == (2, 7): platform += "u" pex_request = TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=Addresses([field_set.address]), entry_point=None, output_filename=pex_filename, platforms=PexPlatforms([platform]), )) pex_result = await Get[TwoStepPex](TwoStepPexFromTargetsRequest, pex_request) input_digest = await Get[Digest](MergeDigests( (pex_result.pex.digest, lambdex_setup.requirements_pex.digest))) # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. lambdex_args = ("build", "-e", field_set.handler.value, pex_filename) process = lambdex_setup.requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./lambdex.pex", pex_args=lambdex_args, input_digest=input_digest, output_files=(pex_filename, ), description=f"Setting up handler in {pex_filename}", ) result = await Get[ProcessResult](Process, process) # Note that the AWS-facing handler function is always lambdex_handler.handler, which # is the wrapper injected by lambdex that manages invocation of the actual handler. return CreatedAWSLambda( digest=result.output_digest, name=pex_filename, runtime=field_set.runtime.value, handler="lambdex_handler.handler", )
async def create_python_binary_run_request( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults) -> RunRequest: entry_point = field_set.entry_point.value if entry_point is None: # TODO: This is overkill? We don't need to hydrate the sources and strip snapshots, # we only need the path relative to the source root. binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources)) stripped_binary_sources = await Get( StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())) entry_point = PythonBinarySources.translate_source_file_to_entry_point( stripped_binary_sources.snapshot.files) if entry_point is None: raise InvalidFieldException( "You must either specify `sources` or `entry_point` for the target " f"{repr(field_set.address)} in order to run it, but both fields were undefined." ) transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address])) output_filename = f"{field_set.address.target_name}.pex" pex_request = Get( Pex, PexFromTargetsRequest( addresses=Addresses([field_set.address]), platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args( python_binary_defaults), include_source_files=False, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True), ) pex, sources = await MultiGet(pex_request, sources_request) merged_digest = await Get( Digest, MergeDigests([pex.digest, sources.source_files.snapshot.digest])) return RunRequest( digest=merged_digest, binary_name=pex.output_filename, prefix_args=("-m", entry_point), env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)}, )
def create_pex_and_get_all_data( self, *, requirements=PexRequirements(), entry_point=None, interpreter_constraints=PexInterpreterConstraints(), platforms=PexPlatforms(), sources: Optional[Digest] = None, additional_inputs: Optional[Digest] = None, additional_pants_args: Tuple[str, ...] = (), additional_pex_args: Tuple[str, ...] = (), ) -> Dict: request = PexRequest( output_filename="test.pex", requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=platforms, entry_point=entry_point, sources=sources, additional_inputs=additional_inputs, additional_args=additional_pex_args, ) pex = self.request_single_product( Pex, Params( request, create_options_bootstrapper(args=[ "--backend-packages2=pants.backend.python", *additional_pants_args ]), ), ) self.scheduler.materialize_directory(DirectoryToMaterialize( pex.digest)) pex_path = os.path.join(self.build_root, "test.pex") with zipfile.ZipFile(pex_path, "r") as zipfp: with zipfp.open("PEX-INFO", "r") as pex_info: pex_info_content = pex_info.readline().decode() pex_list = zipfp.namelist() return { "pex": pex, "local_path": pex_path, "info": json.loads(pex_info_content), "files": pex_list, }
def test_platforms(self) -> None: # We use Python 2.7, rather than Python 3, to ensure that the specified platform is # actually used. platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"]) constraints = PexInterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"]) pex_output = self.create_pex_and_get_all_data( requirements=PexRequirements(["cryptography==2.9"]), platforms=platforms, interpreter_constraints=constraints, ) assert any( "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp for fp in pex_output["files"] ) assert not any("cryptography-2.9-cp27-cp27m-" in fp for fp in pex_output["files"]) assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_output["files"]) # NB: Platforms override interpreter constraints. assert pex_output["info"]["interpreter_constraints"] == []
async def create_python_binary( field_set: PythonBinaryFieldSet) -> CreatedBinary: entry_point = field_set.entry_point.value if entry_point is None: source_files = await Get[SourceFiles](AllSourceFilesRequest( [field_set.sources], strip_source_roots=True)) entry_point = PythonBinarySources.translate_source_file_to_entry_point( source_files.files) output_filename = f"{field_set.address.target_name}.pex" two_step_pex = await Get[TwoStepPex](TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=Addresses([field_set.address]), entry_point=entry_point, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args(), ))) pex = two_step_pex.pex return CreatedBinary(digest=pex.digest, binary_name=pex.output_filename)
def __init__(self, addresses: Addresses, *, output_filename: str, entry_point: Optional[str] = None, platforms: PexPlatforms = PexPlatforms(), additional_args: Iterable[str] = (), additional_requirements: Iterable[str] = (), include_source_files: bool = True, additional_sources: Optional[Digest] = None, additional_inputs: Optional[Digest] = None, description: Optional[str] = None) -> None: self.addresses = addresses self.output_filename = output_filename self.entry_point = entry_point self.platforms = platforms self.additional_args = tuple(additional_args) self.additional_requirements = tuple(additional_requirements) self.include_source_files = include_source_files self.additional_sources = additional_sources self.additional_inputs = additional_inputs self.description = description
def create_pex_and_get_pex_info( self, *, requirements=PexRequirements(), entry_point=None, interpreter_constraints=PexInterpreterConstraints(), platforms=PexPlatforms(), sources: Optional[Digest] = None, additional_pants_args: Tuple[str, ...] = (), additional_pex_args: Tuple[str, ...] = (), ) -> Dict: return cast( Dict, self.create_pex_and_get_all_data( requirements=requirements, entry_point=entry_point, interpreter_constraints=interpreter_constraints, platforms=platforms, sources=sources, additional_pants_args=additional_pants_args, additional_pex_args=additional_pex_args, )["info"], )