async def export_venv(request: ExportedVenvRequest, python_setup: PythonSetup, pex_env: PexEnvironment) -> ExportableData: # Pick a single interpreter for the venv. interpreter_constraints = InterpreterConstraints.create_from_targets( request.targets, python_setup) if not interpreter_constraints: # If there were no targets that defined any constraints, fall back to the global ones. interpreter_constraints = InterpreterConstraints( python_setup.interpreter_constraints) min_interpreter = interpreter_constraints.snap_to_minimum( python_setup.interpreter_universe) if not min_interpreter: raise ExportError( "The following interpreter constraints were computed for all the targets for which " f"export was requested: {interpreter_constraints}. There is no python interpreter " "compatible with these constraints. Please restrict the target set to one that shares " "a compatible interpreter.") venv_pex = await Get( VenvPex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in request.targets), internal_only=True, hardcoded_interpreter_constraints=min_interpreter, ), ) complete_pex_env = pex_env.in_workspace() venv_abspath = os.path.join(complete_pex_env.pex_root, venv_pex.venv_rel_dir) # Run the venv_pex to get the full python version (including patch #), so we # can use it in the symlink name. res = await Get( ProcessResult, VenvPexProcess( venv_pex=venv_pex, description="Create virtualenv", argv=[ "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))" ], input_digest=venv_pex.digest, ), ) py_version = res.stdout.strip().decode() return ExportableData( f"virtualenv for {min_interpreter}", os.path.join("python", "virtualenv"), symlinks=[Symlink(venv_abspath, py_version)], )
async def create_python_repl_request(request: PythonRepl, pex_env: PexEnvironment, python_setup: PythonSetup) -> ReplRequest: validate_compatible_resolve(request.targets, python_setup) interpreter_constraints, transitive_targets = await MultiGet( Get(InterpreterConstraints, InterpreterConstraintsRequest(request.addresses)), Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)), ) requirements_request = Get(Pex, RequirementsPexRequest(request.addresses)) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=True, interpreter_constraints=interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(request.in_chroot( requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [ request.in_chroot(sr) for sr in sources.source_roots ] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": request.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so # that we can get the interpreter constraints for use in local_dists_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( Addresses(tgt.address for tgt in repl.targets), internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": repl.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, python_setup: PythonSetup, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in local_dists_get. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest( [field_set.address], output_filename=f"{field_set.address.target_name}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, resolve_and_lockfile=field_set.resolve.resolve_and_lockfile( python_setup), additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) pex_get = Get(Pex, PexRequest, requirements_pex_request) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests([ pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment) -> RunRequest: run_in_sandbox = field_set.run_in_sandbox.value entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) addresses = [field_set.address] interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest(addresses)) pex_filename = (field_set.address.generated_name.replace(".", "_") if field_set.address.generated_name else field_set.address.target_name) pex_get = Get( Pex, PexFromTargetsRequest( [field_set.address], output_filename=f"{pex_filename}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=interpreter_constraints, sources=sources, ), ) input_digests = [ pex.digest, local_dists.pex.digest, # Note regarding inline mode: You might think that the sources don't need to be copied # into the chroot when using inline sources. But they do, because some of them might be # codegenned, and those won't exist in the inline source tree. Rather than incurring the # complexity of figuring out here which sources were codegenned, we copy everything. # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline # sources will take precedence and their copies in the chroot will be ignored. local_dists.remaining_sources.source_files.snapshot.digest, ] merged_digest = await Get(Digest, MergeDigests(input_digests)) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] # The order here is important: we want the in-repo sources to take precedence over their # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). source_roots = [ *([] if run_in_sandbox else sources.source_roots), *chrooted_source_roots, ] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so # that we can get the interpreter constraints for use in ipython_request/local_dists_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) ipython_request = Get( Pex, PexRequest( output_filename="ipython.pex", main=ipython.main, requirements=ipython.pex_requirements(), interpreter_constraints=requirements_pex_request. interpreter_constraints, internal_only=True, ), ) requirements_pex, sources, ipython_pex = await MultiGet( requirements_request, sources_request, ipython_request) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [tgt.address for tgt in repl.targets], interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests(( requirements_pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ipython_pex.digest, )), ) complete_pex_env = pex_env.in_workspace() args = list( complete_pex_env.create_argv(repl.in_chroot(ipython_pex.name), python=ipython_pex.python)) if ipython.options.ignore_cwd: args.append("--ignore-cwd") chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None), "PEX_PATH": os.pathsep.join([ repl.in_chroot(requirements_pex_request.output_filename), repl.in_chroot(local_dists.pex.name), ]), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)