def create_pex(request: CreatePex, pex_bin: DownloadedPexBin, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform) -> Pex: """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" interpreter_constraint_args = [] for constraint in request.interpreter_constraints: interpreter_constraint_args.extend( ["--interpreter-constraint", constraint]) argv = ["--output-file", request.output_filename] if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) source_dir_name = 'source_files' argv.append(f'--sources-directory={source_dir_name}') sources_digest = request.input_files_digest if request.input_files_digest else EMPTY_DIRECTORY_DIGEST sources_digest_as_subdir = yield Get( Digest, DirectoryWithPrefixToAdd(sources_digest, source_dir_name)) all_inputs = ( pex_bin.directory_digest, sources_digest_as_subdir, ) merged_digest = yield Get(Digest, DirectoriesToMerge(directories=all_inputs)) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building we specify that out PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for out # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". execute_process_request = MultiPlatformExecuteProcessRequest( { (PlatformConstraint(platform.value), PlatformConstraint(platform.value)): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description= f"Create a requirements PEX: {', '.join(request.requirements)}", output_files=(request.output_filename, )) }) result = yield Get(ExecuteProcessResult, MultiPlatformExecuteProcessRequest, execute_process_request) yield Pex(directory_digest=result.output_directory_digest)
def create_requirements_pex(request, pex_bin, python_setup, pex_build_environment, platform): """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" interpreter_search_paths = create_path_env_var( python_setup.interpreter_search_paths) env = { "PATH": interpreter_search_paths, **pex_build_environment.invocation_environment_dict } interpreter_constraint_args = [] for constraint in request.interpreter_constraints: interpreter_constraint_args.extend( ["--interpreter-constraint", constraint]) # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. argv = [ "python", f"./{pex_bin.executable}", "--output-file", request.output_filename ] if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) # NOTE # PEX outputs are platform dependent so in order to get a PEX that we can use locally, without cross-building # we specify that out PEX command be run on the current local platform. When we support cross-building # through CLI flags we can configure requests that build a PEX for out local platform that are # able to execute on a different platform, but for now in order to guarantee correct build we need # to restrict this command to execute on the same platform type that the output is intended for. # The correct way to interpret the keys (execution_platform_constraint, target_platform_constraint) # of this dictionary is "The output of this command is intended for `target_platform_constraint` iff # it is run on `execution_platform_constraint`". execute_process_request = MultiPlatformExecuteProcessRequest( { (PlatformConstraint(platform.value), PlatformConstraint(platform.value)): ExecuteProcessRequest( argv=tuple(argv), env=env, input_files=pex_bin.directory_digest, description= f"Create a requirements PEX: {', '.join(request.requirements)}", output_files=(request.output_filename, )) }) result = yield Get(ExecuteProcessResult, MultiPlatformExecuteProcessRequest, execute_process_request) yield RequirementsPex(directory_digest=result.output_directory_digest)
def _deserialized_mapping( self, ) -> Dict[PlatformConstraint, ToolForPlatform]: deserialized: Dict[PlatformConstraint, ToolForPlatform] = {} for platform_constraint, (version, fingerprint, size_bytes) in self.version_digest_mapping: deserialized[PlatformConstraint( platform_constraint)] = ToolForPlatform( version=ToolVersion(version), digest=Digest(fingerprint, size_bytes), ) return deserialized
def __new__(cls, request_dict): if len(request_dict) == 0: raise cls.make_type_error("At least one platform constrained ExecuteProcessRequest must be passed.") # validate the platform constraints using the platforms enum an flatten the keys. validated_constraints = tuple( constraint.value for pair in request_dict.keys() for constraint in pair if PlatformConstraint(constraint.value) ) if len({req.description for req in request_dict.values()}) != 1: raise ValueError(f"The `description` of all execute_process_requests in a {cls.__name__} must be identical.") return super().__new__( cls, validated_constraints, tuple(request_dict.values()) )
def __init__( self, request_dict: Dict[Tuple[PlatformConstraint, PlatformConstraint], Process], ) -> None: if len(request_dict) == 0: raise ValueError("At least one platform constrained Process must be passed.") validated_constraints = tuple( constraint.value for pair in request_dict.keys() for constraint in pair if PlatformConstraint(constraint.value) ) if len({req.description for req in request_dict.values()}) != 1: raise ValueError( f"The `description` of all processes in a {MultiPlatformProcess.__name__} must be identical." ) self.platform_constraints = validated_constraints self.processes = tuple(request_dict.values())
async def create_pex( request: CreatePex, pex_bin: DownloadedPexBin, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform, ) -> Pex: """Returns a PEX with the given requirements, optional entry point, optional interpreter constraints, and optional requirement constraints.""" argv = [ "--output-file", request.output_filename, *request.interpreter_constraints.generate_pex_arg_list(), *request.additional_args, ] if python_setup.resolver_jobs: argv.extend(["--jobs", python_setup.resolver_jobs]) if python_setup.manylinux: argv.extend(["--manylinux", python_setup.manylinux]) else: argv.append("--no-manylinux") if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) if python_setup.requirement_constraints is not None: argv.extend(["--constraints", python_setup.requirement_constraints]) source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") argv.extend(request.requirements.requirements) constraint_file_snapshot = EMPTY_SNAPSHOT if python_setup.requirement_constraints is not None: constraint_file_snapshot = await Get[Snapshot](PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", )) sources_digest = (request.input_files_digest if request.input_files_digest else EMPTY_DIRECTORY_DIGEST) sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd( sources_digest, source_dir_name)) merged_digest = await Get[Digest](DirectoriesToMerge(directories=( pex_bin.directory_digest, sources_digest_as_subdir, constraint_file_snapshot.directory_digest, ))) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building, we specify that our PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for our # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". execute_process_request = MultiPlatformExecuteProcessRequest({ ( PlatformConstraint(platform.value), PlatformConstraint(platform.value), ): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description= f"Create a requirements PEX: {', '.join(request.requirements.requirements)}", output_files=(request.output_filename, ), ) }) result = await Get[ExecuteProcessResult]( MultiPlatformExecuteProcessRequest, execute_process_request) return Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)
async def create_pex( request: PexRequest, pex_bin: DownloadedPexBin, python_setup: PythonSetup, python_repos: PythonRepos, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform, log_level: LogLevel, ) -> Pex: """Returns a PEX with the given requirements, optional entry point, optional interpreter constraints, and optional requirement constraints.""" argv = [ "--output-file", request.output_filename, *request.interpreter_constraints.generate_pex_arg_list(), # NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes` # including PyPI, which will override `--no-pypi` and result in using PyPI in the default # case. Why set `--no-pypi`, then? We need to do this so that # `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI. "--no-pypi", *(f"--index={index}" for index in python_repos.indexes), *(f"--repo={repo}" for repo in python_repos.repos), *request.additional_args, ] pex_debug = PexDebug(log_level) argv.extend(pex_debug.iter_pex_args()) if python_setup.resolver_jobs: argv.extend(["--jobs", python_setup.resolver_jobs]) if python_setup.manylinux: argv.extend(["--manylinux", python_setup.manylinux]) else: argv.append("--no-manylinux") if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) if python_setup.requirement_constraints is not None: argv.extend(["--constraints", python_setup.requirement_constraints]) source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") argv.extend(request.requirements.requirements) constraint_file_snapshot = EMPTY_SNAPSHOT if python_setup.requirement_constraints is not None: constraint_file_snapshot = await Get[Snapshot](PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", )) sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd( request.sources or EMPTY_DIRECTORY_DIGEST, source_dir_name)) additional_inputs_digest = request.additional_inputs or EMPTY_DIRECTORY_DIGEST merged_digest = await Get[Digest](DirectoriesToMerge(directories=( pex_bin.directory_digest, sources_digest_as_subdir, additional_inputs_digest, constraint_file_snapshot.directory_digest, ))) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building, we specify that our PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for our # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". description = request.description if description is None: if request.requirements.requirements: description = f"Resolving {', '.join(request.requirements.requirements)}" else: description = f"Building PEX" execute_process_request = MultiPlatformExecuteProcessRequest({ ( PlatformConstraint(platform.value), PlatformConstraint(platform.value), ): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description=description, output_files=(request.output_filename, ), ) }) result = await Get[ExecuteProcessResult]( MultiPlatformExecuteProcessRequest, execute_process_request) if pex_debug.might_log: lines = result.stderr.decode().splitlines() if lines: pex_debug.log( f"Debug output from Pex for: {execute_process_request}") for line in lines: pex_debug.log(line) return Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)
async def create_pex( request: PexRequest, python_setup: PythonSetup, python_repos: PythonRepos, platform: Platform, pex_runtime_environment: PexRuntimeEnvironment, ) -> Pex: """Returns a PEX with the given settings.""" argv = [ "--output-file", request.output_filename, # NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes` # including PyPI, which will override `--no-pypi` and result in using PyPI in the default # case. Why set `--no-pypi`, then? We need to do this so that # `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI. "--no-pypi", *(f"--index={index}" for index in python_repos.indexes), *(f"--repo={repo}" for repo in python_repos.repos), *request.additional_args, ] if request.internal_only: # This will result in a faster build, but worse compatibility at runtime. argv.append("--use-first-matching-interpreter") # NB: If `--platform` is specified, this signals that the PEX should not be built locally. # `--interpreter-constraint` only makes sense in the context of building locally. These two # flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957. if request.platforms: # TODO(#9560): consider validating that these platforms are valid with the interpreter # constraints. argv.extend(request.platforms.generate_pex_arg_list()) else: argv.extend(request.interpreter_constraints.generate_pex_arg_list()) argv.append("--no-emit-warnings") verbosity = pex_runtime_environment.verbosity if verbosity > 0: argv.append(f"-{'v' * verbosity}") if python_setup.resolver_jobs: argv.extend(["--jobs", str(python_setup.resolver_jobs)]) if python_setup.manylinux: argv.extend(["--manylinux", python_setup.manylinux]) else: argv.append("--no-manylinux") if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) if python_setup.requirement_constraints is not None: argv.extend(["--constraints", python_setup.requirement_constraints]) source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") argv.extend(request.requirements) constraint_file_digest = EMPTY_DIGEST if python_setup.requirement_constraints is not None: constraint_file_digest = await Get( Digest, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", ), ) sources_digest_as_subdir = await Get( Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name)) additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST merged_digest = await Get( Digest, MergeDigests(( sources_digest_as_subdir, additional_inputs_digest, constraint_file_digest, )), ) description = request.description if description is None: if request.requirements: description = ( f"Building {request.output_filename} with " f"{pluralize(len(request.requirements), 'requirement')}: " f"{', '.join(request.requirements)}") else: description = f"Building {request.output_filename}" process = await Get( Process, PexCliProcess( argv=argv, additional_input_digest=merged_digest, description=description, output_files=[request.output_filename], ), ) # NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally # without cross-building, we specify that our PEX command should be run on the current local # platform. result = await Get( ProcessResult, MultiPlatformProcess({ (PlatformConstraint(platform.value), PlatformConstraint(platform.value)): process }), ) if verbosity > 0: log_output = result.stderr.decode() if log_output: logger.info("%s", log_output) return Pex( digest=result.output_digest, name=request.output_filename, internal_only=request.internal_only, )