async def export_venv(request: ExportedVenvRequest, python_setup: PythonSetup,
pex_env: PexEnvironment) -> ExportableData:
# Pick a single interpreter for the venv.
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.targets, python_setup)
if not interpreter_constraints:
# If there were no targets that defined any constraints, fall back to the global ones.
interpreter_constraints = InterpreterConstraints(
python_setup.interpreter_constraints)
min_interpreter = interpreter_constraints.snap_to_minimum(
python_setup.interpreter_universe)
if not min_interpreter:
raise ExportError(
"The following interpreter constraints were computed for all the targets for which "
f"export was requested: {interpreter_constraints}. There is no python interpreter "
"compatible with these constraints. Please restrict the target set to one that shares "
"a compatible interpreter.")
venv_pex = await Get(
VenvPex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in request.targets),
internal_only=True,
hardcoded_interpreter_constraints=min_interpreter,
),
)
complete_pex_env = pex_env.in_workspace()
venv_abspath = os.path.join(complete_pex_env.pex_root,
venv_pex.venv_rel_dir)
# Run the venv_pex to get the full python version (including patch #), so we
# can use it in the symlink name.
res = await Get(
ProcessResult,
VenvPexProcess(
venv_pex=venv_pex,
description="Create virtualenv",
argv=[
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"
],
input_digest=venv_pex.digest,
),
)
py_version = res.stdout.strip().decode()
return ExportableData(
f"virtualenv for {min_interpreter}",
os.path.join("python", "virtualenv"),
symlinks=[Symlink(venv_abspath, py_version)],
)
async def create_python_repl_request(request: PythonRepl,
pex_env: PexEnvironment,
python_setup: PythonSetup) -> ReplRequest:
validate_compatible_resolve(request.targets, python_setup)
interpreter_constraints, transitive_targets = await MultiGet(
Get(InterpreterConstraints,
InterpreterConstraintsRequest(request.addresses)),
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
)
requirements_request = Get(Pex, RequirementsPexRequest(request.addresses))
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=True,
interpreter_constraints=interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(request.in_chroot(
requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [
request.in_chroot(sr) for sr in sources.source_roots
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
request.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so
# that we can get the interpreter constraints for use in local_dists_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
Addresses(tgt.address for tgt in repl.targets),
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
repl.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
python_setup: PythonSetup,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in local_dists_get.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{field_set.address.target_name}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
python_setup),
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
pex_get = Get(Pex, PexRequest, requirements_pex_request)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests([
pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment) -> RunRequest:
run_in_sandbox = field_set.run_in_sandbox.value
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
addresses = [field_set.address]
interpreter_constraints = await Get(
InterpreterConstraints, InterpreterConstraintsRequest(addresses))
pex_filename = (field_set.address.generated_name.replace(".", "_")
if field_set.address.generated_name else
field_set.address.target_name)
pex_get = Get(
Pex,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{pex_filename}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
input_digests = [
pex.digest,
local_dists.pex.digest,
# Note regarding inline mode: You might think that the sources don't need to be copied
# into the chroot when using inline sources. But they do, because some of them might be
# codegenned, and those won't exist in the inline source tree. Rather than incurring the
# complexity of figuring out here which sources were codegenned, we copy everything.
# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
# sources will take precedence and their copies in the chroot will be ignored.
local_dists.remaining_sources.source_files.snapshot.digest,
]
merged_digest = await Get(Digest, MergeDigests(input_digests))
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
# The order here is important: we want the in-repo sources to take precedence over their
# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
source_roots = [
*([] if run_in_sandbox else sources.source_roots),
*chrooted_source_roots,
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so
# that we can get the interpreter constraints for use in ipython_request/local_dists_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
ipython_request = Get(
Pex,
PexRequest(
output_filename="ipython.pex",
main=ipython.main,
requirements=ipython.pex_requirements(),
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
internal_only=True,
),
)
requirements_pex, sources, ipython_pex = await MultiGet(
requirements_request, sources_request, ipython_request)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[tgt.address for tgt in repl.targets],
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests((
requirements_pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
ipython_pex.digest,
)),
)
complete_pex_env = pex_env.in_workspace()
args = list(
complete_pex_env.create_argv(repl.in_chroot(ipython_pex.name),
python=ipython_pex.python))
if ipython.options.ignore_cwd:
args.append("--ignore-cwd")
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
"PEX_PATH":
os.pathsep.join([
repl.in_chroot(requirements_pex_request.output_filename),
repl.in_chroot(local_dists.pex.name),
]),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)