async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process: pex = request.pex argv = pex_environment.create_argv(pex.name, *request.argv, python=pex.python) env = { **pex_environment.environment_dict(python_configured=pex.python is not None), **(request.extra_env or {}), } input_digest = ( await Get(Digest, MergeDigests((pex.digest, request.input_digest))) if request.input_digest else pex.digest ) return Process( argv, description=request.description, level=request.level, input_digest=input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches=pex_environment.append_only_caches(), timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, cache_scope=request.cache_scope, )
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest: requirements_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) requirements_pex, sources = await MultiGet(requirements_request, sources_request) merged_digest = await Get( Digest, MergeDigests( (requirements_pex.digest, sources.source_files.snapshot.digest))) args = pex_env.create_argv(repl.in_chroot(requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process: argv = pex_environment.create_argv( f"./{request.pex.name}", *request.argv, python=request.pex.python, ) env = { **pex_environment.environment_dict(python_configured=request.pex.python is not None), **(request.extra_env or {}), } process = Process( argv, description=request.description, level=request.level, input_digest=request.input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, ) return await Get( Process, UncacheableProcess(process)) if request.uncacheable else process
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in ipython_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) ipython_request = Get( Pex, PexRequest( output_filename="ipython.pex", main=ipython.main, requirements=PexRequirements(ipython.all_requirements), interpreter_constraints=requirements_pex_request. interpreter_constraints, internal_only=True, ), ) requirements_pex, sources, ipython_pex = await MultiGet( requirements_request, sources_request, ipython_request) merged_digest = await Get( Digest, MergeDigests( (requirements_pex.digest, sources.source_files.snapshot.digest, ipython_pex.digest)), ) args = list( pex_env.create_argv(repl.in_chroot(ipython_pex.name), python=ipython_pex.python)) if ipython.options.ignore_cwd: args.append("--ignore-cwd") chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=ipython_pex.python is not None), "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_venv_pex_process( request: VenvPexProcess, pex_environment: PexEnvironment ) -> Process: venv_pex = request.venv_pex pex_bin = ( os.path.relpath(venv_pex.pex.argv0, request.working_directory) if request.working_directory else venv_pex.pex.argv0 ) argv = (pex_bin, *request.argv) input_digest = ( await Get(Digest, MergeDigests((venv_pex.digest, request.input_digest))) if request.input_digest else venv_pex.digest ) return Process( argv=argv, description=request.description, level=request.level, input_digest=input_digest, working_directory=request.working_directory, env=request.extra_env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches=pex_environment.in_sandbox( working_directory=request.working_directory ).append_only_caches, timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, cache_scope=request.cache_scope, )
def _create_venv_script( self, bash: BashBinary, pex_environment: PexEnvironment, *, script_path: PurePath, venv_executable: PurePath, ) -> VenvScript: env_vars = ( f"{name}={shlex.quote(value)}" for name, value in pex_environment.environment_dict(python_configured=True).items() ) target_venv_executable = shlex.quote(str(venv_executable)) venv_dir = shlex.quote(str(self.venv_dir)) execute_pex_args = " ".join( shlex.quote(arg) for arg in pex_environment.create_argv(self.pex.name, python=self.pex.python) ) script = dedent( f"""\ #!{bash.path} set -euo pipefail export {" ".join(env_vars)} # Let PEX_TOOLS invocations pass through to the original PEX file since venvs don't come # with tools support. if [ -n "${{PEX_TOOLS:-}}" ]; then exec {execute_pex_args} "$@" fi # If the seeded venv has been removed from the PEX_ROOT, we re-seed from the original # `--venv` mode PEX file. if [ ! -e {target_venv_executable} ]; then rm -rf {venv_dir} || true PEX_INTERPRETER=1 {execute_pex_args} -c '' fi exec {target_venv_executable} "$@" """ ) return VenvScript( script=Script(script_path), content=FileContent(path=str(script_path), content=script.encode(), is_executable=True), )
def create( cls, pex_environment: PexEnvironment, pex: Pex, venv_rel_dir: PurePath ) -> VenvScriptWriter: # N.B.: We don't know the working directory that will be used in any given # invocation of the venv scripts; so we deal with working_directory inside the scripts # themselves by absolutifying all relevant paths at runtime. complete_pex_env = pex_environment.in_sandbox(working_directory=None) venv_dir = complete_pex_env.pex_root / venv_rel_dir return cls(complete_pex_env=complete_pex_env, pex=pex, venv_dir=venv_dir)
def create(cls, pex_environment: PexEnvironment, pex: Pex, venv_rel_dir: PurePath) -> VenvScriptWriter: # N.B.: We don't know the working directory that will be used in any given # invocation of the venv scripts; so we deal with working_directory once in an # `adjust_relative_paths` function inside the script to save rule authors from having to do # CWD offset math in every rule for all the relative paths their process depends on. complete_pex_env = pex_environment.in_sandbox(working_directory=None) venv_dir = complete_pex_env.pex_root / venv_rel_dir return cls(complete_pex_env=complete_pex_env, pex=pex, venv_dir=venv_dir)
async def export_venv(request: ExportedVenvRequest, python_setup: PythonSetup, pex_env: PexEnvironment) -> ExportableData: # Pick a single interpreter for the venv. interpreter_constraints = InterpreterConstraints.create_from_targets( request.targets, python_setup) if not interpreter_constraints: # If there were no targets that defined any constraints, fall back to the global ones. interpreter_constraints = InterpreterConstraints( python_setup.interpreter_constraints) min_interpreter = interpreter_constraints.snap_to_minimum( python_setup.interpreter_universe) if not min_interpreter: raise ExportError( "The following interpreter constraints were computed for all the targets for which " f"export was requested: {interpreter_constraints}. There is no python interpreter " "compatible with these constraints. Please restrict the target set to one that shares " "a compatible interpreter.") venv_pex = await Get( VenvPex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in request.targets), internal_only=True, hardcoded_interpreter_constraints=min_interpreter, ), ) complete_pex_env = pex_env.in_workspace() venv_abspath = os.path.join(complete_pex_env.pex_root, venv_pex.venv_rel_dir) # Run the venv_pex to get the full python version (including patch #), so we # can use it in the symlink name. res = await Get( ProcessResult, VenvPexProcess( venv_pex=venv_pex, description="Create virtualenv", argv=[ "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))" ], input_digest=venv_pex.digest, ), ) py_version = res.stdout.strip().decode() return ExportableData( f"virtualenv for {min_interpreter}", os.path.join("python", "virtualenv"), symlinks=[Symlink(venv_abspath, py_version)], )
async def create_python_repl_request(request: PythonRepl, pex_env: PexEnvironment, python_setup: PythonSetup) -> ReplRequest: validate_compatible_resolve(request.targets, python_setup) interpreter_constraints, transitive_targets = await MultiGet( Get(InterpreterConstraints, InterpreterConstraintsRequest(request.addresses)), Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)), ) requirements_request = Get(Pex, RequirementsPexRequest(request.addresses)) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=True, interpreter_constraints=interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(request.in_chroot( requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [ request.in_chroot(sr) for sr in sources.source_roots ] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": request.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so # that we can get the interpreter constraints for use in local_dists_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( Addresses(tgt.address for tgt in repl.targets), internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": repl.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment) -> RunRequest: run_in_sandbox = field_set.run_in_sandbox.value entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) addresses = [field_set.address] interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest(addresses)) pex_filename = (field_set.address.generated_name.replace(".", "_") if field_set.address.generated_name else field_set.address.target_name) pex_get = Get( Pex, PexFromTargetsRequest( [field_set.address], output_filename=f"{pex_filename}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=interpreter_constraints, sources=sources, ), ) input_digests = [ pex.digest, local_dists.pex.digest, # Note regarding inline mode: You might think that the sources don't need to be copied # into the chroot when using inline sources. But they do, because some of them might be # codegenned, and those won't exist in the inline source tree. Rather than incurring the # complexity of figuring out here which sources were codegenned, we copy everything. # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline # sources will take precedence and their copies in the chroot will be ignored. local_dists.remaining_sources.source_files.snapshot.digest, ] merged_digest = await Get(Digest, MergeDigests(input_digests)) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] # The order here is important: we want the in-repo sources to take precedence over their # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). source_roots = [ *([] if run_in_sandbox else sources.source_roots), *chrooted_source_roots, ] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( pex_binary_defaults), internal_only=True, # Note that the entry point file is not in the PEX itself. It's loaded by setting # `PEX_EXTRA_SYS_PATH`. # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=entry_point.val, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process( request: PexCliProcess, pex_binary: PexBinary, pex_env: PexEnvironment, python_native_code: PythonNativeCode, global_options: GlobalOptions, pex_runtime_env: PexRuntimeEnvironment, ) -> Process: tmpdir = ".tmp" gets: List[Get] = [ Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)), Get(Digest, CreateDigest([Directory(tmpdir)])), ] cert_args = [] # The certs file will typically not be in the repo, so we can't digest it via a PathGlobs. # Instead we manually create a FileContent for it. if global_options.options.ca_certs_path: ca_certs_content = Path(global_options.options.ca_certs_path).read_bytes() chrooted_ca_certs_path = os.path.basename(global_options.options.ca_certs_path) gets.append( Get( Digest, CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)), ) ) cert_args = ["--cert", chrooted_ca_certs_path] downloaded_pex_bin, *digests_to_merge = await MultiGet(gets) digests_to_merge.append(downloaded_pex_bin.digest) if request.additional_input_digest: digests_to_merge.append(request.additional_input_digest) input_digest = await Get(Digest, MergeDigests(digests_to_merge)) pex_root_path = ".cache/pex_root" argv = [ downloaded_pex_bin.exe, *cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths), "--pex-root", pex_root_path, # Ensure Pex and its subprocesses create temporary files in the the process execution # sandbox. It may make sense to do this generally for Processes, but in the short term we # have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to # perform. Making the TMPDIR local to the sandbox allows control via # --local-execution-root-dir for the local case and should work well with remote cases where # a remoting implementation has to allow for processes producing large binaries in a # sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via # environment variable allows Pex to absolutize the path ensuring subprocesses that change # CWD can find the TMPDIR. "--tmpdir", tmpdir, ] if pex_runtime_env.verbosity > 0: argv.append(f"-{'v' * pex_runtime_env.verbosity}") # NB: This comes at the end of the argv because the request may use `--` passthrough args, # which must come at the end. argv.extend(request.argv) normalized_argv = pex_env.create_argv(*argv, python=request.python) env = { **pex_env.environment_dict(python_configured=request.python is not None), **python_native_code.environment_dict, **(request.extra_env or {}), } return Process( normalized_argv, description=request.description, input_digest=input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches={"pex_root": pex_root_path}, level=request.level, cache_scope=request.cache_scope, )
async def setup_pex_cli_process( request: PexCliProcess, pex_pex: PexPEX, pex_env: PexEnvironment, python_native_code: PythonNativeCode, global_options: GlobalOptions, pex_runtime_env: PexRuntimeEnvironment, ) -> Process: tmpdir = ".tmp" gets: List[Get] = [Get(Digest, CreateDigest([Directory(tmpdir)]))] # The certs file will typically not be in the repo, so we can't digest it via a PathGlobs. # Instead we manually create a FileContent for it. cert_args = [] if global_options.ca_certs_path: ca_certs_content = Path(global_options.ca_certs_path).read_bytes() chrooted_ca_certs_path = os.path.basename(global_options.ca_certs_path) gets.append( Get( Digest, CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)), ) ) cert_args = ["--cert", chrooted_ca_certs_path] digests_to_merge = [pex_pex.digest] digests_to_merge.extend(await MultiGet(gets)) if request.additional_input_digest: digests_to_merge.append(request.additional_input_digest) input_digest = await Get(Digest, MergeDigests(digests_to_merge)) global_args = [ # Ensure Pex and its subprocesses create temporary files in the the process execution # sandbox. It may make sense to do this generally for Processes, but in the short term we # have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to # perform. Making the TMPDIR local to the sandbox allows control via # --local-execution-root-dir for the local case and should work well with remote cases where # a remoting implementation has to allow for processes producing large binaries in a # sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via # environment variable allows Pex to absolutize the path ensuring subprocesses that change # CWD can find the TMPDIR. "--tmpdir", tmpdir, ] if request.concurrency_available > 0: global_args.extend(["--jobs", "{pants_concurrency}"]) if pex_runtime_env.verbosity > 0: global_args.append(f"-{'v' * pex_runtime_env.verbosity}") resolve_args = ( [*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)] if request.set_resolve_args else [] ) args = [ *global_args, *request.subcommand, *resolve_args, # NB: This comes at the end because it may use `--` passthrough args, # which must come at # the end. *request.extra_args, ] complete_pex_env = pex_env.in_sandbox(working_directory=None) normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python) env = { **complete_pex_env.environment_dict(python_configured=request.python is not None), **python_native_code.environment_dict, **(request.extra_env or {}), # If a subcommand is used, we need to use the `pex3` console script. **({"PEX_SCRIPT": "pex3"} if request.subcommand else {}), } return Process( normalized_argv, description=request.description, input_digest=input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches=complete_pex_env.append_only_caches, level=request.level, concurrency_available=request.concurrency_available, cache_scope=request.cache_scope, )
async def create_python_binary_run_request( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point = field_set.entry_point.value if entry_point is None: binary_source_paths = await Get( Paths, PathGlobs, field_set.sources.path_globs(FilesNotFoundBehavior.error)) if len(binary_source_paths.files) != 1: raise InvalidFieldException( "No `entry_point` was set for the target " f"{repr(field_set.address)}, so it must have exactly one source, but it has " f"{len(binary_source_paths.files)}") entry_point_path = binary_source_paths.files[0] source_root = await Get( SourceRoot, SourceRootRequest, SourceRootRequest.for_file(entry_point_path), ) entry_point = PythonBinarySources.translate_source_file_to_entry_point( os.path.relpath(entry_point_path, source_root.path)) transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address])) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( python_binary_defaults), internal_only=True, # Note that the entry point file is not in the Pex itself, but on the # PEX_PATH. This works fine! entry_point=entry_point, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc, grpc_python_plugin: GrpcPythonPlugin, python_protobuf_subsystem: PythonProtobufSubsystem, python_protobuf_mypy_plugin: PythonProtobufMypyPlugin, pex_environment: PexEnvironment, ) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest(tgt[ProtobufSourceField] for tgt in transitive_targets.closure if tgt.has_field(ProtobufSourceField)), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSourceField]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) protoc_gen_mypy_script = "protoc-gen-mypy" protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc" mypy_pex = None mypy_request = PexRequest( output_filename="mypy_protobuf.pex", internal_only=True, requirements=python_protobuf_mypy_plugin.pex_requirements(), interpreter_constraints=python_protobuf_mypy_plugin. interpreter_constraints, ) if python_protobuf_subsystem.mypy_plugin: mypy_pex = await Get( VenvPex, VenvPexRequest(bin_names=[protoc_gen_mypy_script], pex_request=mypy_request), ) if request.protocol_target.get(ProtobufGrpcToggleField).value: mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex) # In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above. if any(dist_info.project_name == "mypy-protobuf" and dist_info.version.major >= 2 for dist_info in mypy_info): # TODO: Use `pex_path` once VenvPex stores a Pex field. mypy_pex = await Get( VenvPex, VenvPexRequest( bin_names=[ protoc_gen_mypy_script, protoc_gen_mypy_grpc_script ], pex_request=mypy_request, ), ) downloaded_grpc_plugin = (await Get( DownloadedExternalTool, ExternalToolRequest, grpc_python_plugin.get_request(Platform.current), ) if request.protocol_target.get(ProtobufGrpcToggleField).value else None) unmerged_digests = [ all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, ] if mypy_pex: unmerged_digests.append(mypy_pex.digest) if downloaded_grpc_plugin: unmerged_digests.append(downloaded_grpc_plugin.digest) input_digest = await Get(Digest, MergeDigests(unmerged_digests)) argv = [downloaded_protoc_binary.exe, "--python_out", output_dir] if mypy_pex: argv.extend([ f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}", "--mypy_out", output_dir, ]) if downloaded_grpc_plugin: argv.extend([ f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}", "--grpc_out", output_dir ]) if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin: argv.extend([ f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}", "--mypy_grpc_out", output_dir, ]) argv.extend(target_sources_stripped.snapshot.files) result = await Get( ProcessResult, Process( argv, input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), append_only_caches=pex_environment.in_sandbox( working_directory=None).append_only_caches, ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_source's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def create_venv_pex(request: VenvPexRequest, bash: BashBinary, pex_environment: PexEnvironment) -> VenvPex: # VenvPex is motivated by improving performance of Python tools by eliminating traditional PEX # file startup overhead. # # To achieve the minimal overhead (on the order of 1ms) we discard: # 1. Using Pex default mode: # Although this does reduce initial tool execution overhead, it still leaves a minimum # O(100ms) of overhead per subsequent tool invocation. Fundamentally, Pex still needs to # execute its `sys.path` isolation bootstrap code in this case. # 2. Using the Pex `venv` tool: # The idea here would be to create a tool venv as a Process output and then use the tool # venv as an input digest for all tool invocations. This was tried and netted ~500ms of # overhead over raw venv use. # # Instead we use Pex's `--venv` mode. In this mode you can run the Pex file and it will create a # venv on the fly in the PEX_ROOT as needed. Since the PEX_ROOT is a named_cache, we avoid the # digest materialization overhead present in 2 above. Since the venv is naturally isolated we # avoid the `sys.path` isolation overhead of Pex itself present in 1 above. # # This does leave O(50ms) of overhead though for the PEX bootstrap code to detect an already # created venv in the PEX_ROOT and re-exec into it. To eliminate this overhead we execute the # `pex` venv script in the PEX_ROOT directly. This is not robust on its own though, since the # named caches store might be pruned at any time. To guard against that case we introduce a shim # bash script that checks to see if the `pex` venv script exists in the PEX_ROOT and re-creates # the PEX_ROOT venv if not. Using the shim script to run Python tools gets us down to the ~1ms # of overhead we currently enjoy. pex_request = request.pex_request seeded_venv_request = dataclasses.replace( pex_request, additional_args=pex_request.additional_args + ( "--venv", "--seed", "verbose", pex_environment.venv_site_packages_copies_option( use_copies=request.site_packages_copies), ), ) venv_pex_result = await Get(BuildPexResult, PexRequest, seeded_venv_request) # Pex verbose --seed mode outputs the absolute path of the PEX executable as well as the # absolute path of the PEX_ROOT. In the --venv case this is the `pex` script in the venv root # directory. seed_info = json.loads(venv_pex_result.result.stdout.decode()) abs_pex_root = PurePath(seed_info["pex_root"]) abs_pex_path = PurePath(seed_info["pex"]) venv_rel_dir = abs_pex_path.relative_to(abs_pex_root).parent venv_script_writer = VenvScriptWriter.create( pex_environment=pex_environment, pex=venv_pex_result.create_pex(), venv_rel_dir=venv_rel_dir) pex = venv_script_writer.exe(bash) python = venv_script_writer.python(bash) scripts = { bin_name: venv_script_writer.bin(bash, bin_name) for bin_name in request.bin_names } scripts_digest = await Get( Digest, CreateDigest(( pex.content, python.content, *(venv_script.content for venv_script in scripts.values()), )), ) input_digest = await Get( Digest, MergeDigests((venv_script_writer.pex.digest, scripts_digest))) return VenvPex( digest=input_digest, pex_filename=venv_pex_result.pex_filename, pex=pex.script, python=python.script, bin=FrozenDict((bin_name, venv_script.script) for bin_name, venv_script in scripts.items()), venv_rel_dir=venv_rel_dir.as_posix(), )
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, python_setup: PythonSetup, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in local_dists_get. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest( [field_set.address], output_filename=f"{field_set.address.target_name}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, resolve_and_lockfile=field_set.resolve.resolve_and_lockfile( python_setup), additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) pex_get = Get(Pex, PexRequest, requirements_pex_request) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests([ pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process( request: PexCliProcess, pex_binary: PexBinary, pex_env: PexEnvironment, python_native_code: PythonNativeCode, global_options: GlobalOptions, ) -> Process: tmpdir = ".tmp" gets: List[Get] = [ Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)), Get(Digest, CreateDigest([Directory(f"{tmpdir}/.reserve")])), ] cert_args = [] # The certs file will typically not be in the repo, so we can't digest it via a PathGlobs. # Instead we manually create a FileContent for it. if global_options.options.ca_certs_path: ca_certs_content = Path( global_options.options.ca_certs_path).read_bytes() chrooted_ca_certs_path = os.path.basename( global_options.options.ca_certs_path) gets.append( Get( Digest, CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content), )), )) cert_args = ["--cert", chrooted_ca_certs_path] downloaded_pex_bin, *digests_to_merge = await MultiGet(gets) digests_to_merge.append(downloaded_pex_bin.digest) if request.additional_input_digest: digests_to_merge.append(request.additional_input_digest) input_digest = await Get(Digest, MergeDigests(digests_to_merge)) pex_root_path = ".cache/pex_root" argv = pex_env.create_argv( downloaded_pex_bin.exe, *request.argv, *cert_args, "--pex-root", pex_root_path, python=request.python, ) env = { # Ensure Pex and its subprocesses create temporary files in the the process execution # sandbox. It may make sense to do this generally for Processes, but in the short term we # have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to # perform. Making the TMPDIR local to the sandbox allows control via # --local-execution-root-dir for the local case and should work well with remote cases where # a remoting implementation has to allow for processes producing large binaries in a # sandbox to support reasonable workloads. "TMPDIR": tmpdir, **pex_env.environment_dict(python_configured=request.python is not None), **python_native_code.environment_dict, **(request.extra_env or {}), } return Process( argv, description=request.description, input_digest=input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches={"pex_root": pex_root_path}, level=request.level, )