def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# Don't read pip configuration files like `~/.config/pip/pip.conf`.
"--isolated",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to prompt
# and we'd also like to know about this since it should never occur. We leverage the pip
# global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.
"--exists-action",
"a",
]
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
pex_verbosity = ENV.PEX_VERBOSE
pip_verbosity = pex_verbosity // 3
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + args
with ENV.strip().patch(PEX_ROOT=cache or ENV.PEX_ROOT,
PEX_VERBOSE=str(pex_verbosity)) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".
format(pythonpath),
V=3)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(command=pip.cmdline(command),
process=pip.run(args=command, env=env, blocking=False))
def _spawn_pip_isolated_job(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
pip_verbosity=0, # type: int
**popen_kwargs # type: Any
):
# type: (...) -> Job
command, process = self._spawn_pip_isolated(
args,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=interpreter,
pip_verbosity=pip_verbosity,
**popen_kwargs)
return Job(command=command, process=process)
def spawn_python_job(args,
env=None,
interpreter=None,
expose=None,
pythonpath=None,
**subprocess_kwargs):
"""Spawns a python job.
:param args: The arguments to pass to the python interpreter.
:type args: list of str
:param env: The environment to spawn the python interpreter process in. Defaults to the ambient
environment.
:type env: dict of (str, str)
:param interpreter: The interpreter to use to spawn the python job. Defaults to the current
interpreter.
:type interpreter: :class:`PythonInterpreter`
:param expose: The names of any vendored distributions to expose to the spawned python process.
These will be appended to `pythonpath` if passed.
:type expose: list of str
:param pythonpath: The PYTHONPATH to expose to the spawned python process. These will be
pre-pended to the `expose` path if passed.
:type pythonpath: list of str
:param subprocess_kwargs: Any additional :class:`subprocess.Popen` kwargs to pass through.
:returns: A job handle to the spawned python process.
:rtype: :class:`Job`
"""
pythonpath = list(pythonpath or ())
if expose:
subprocess_env = (env or os.environ).copy()
# In order to expose vendored distributions with their un-vendored import paths in-tact, we
# need to set `__PEX_UNVENDORED__`. See: vendor.__main__.ImportRewriter._modify_import.
subprocess_env['__PEX_UNVENDORED__'] = '1'
pythonpath.extend(third_party.expose(expose))
else:
subprocess_env = env
interpreter = interpreter or PythonInterpreter.get()
cmd, process = interpreter.open_process(args=args,
pythonpath=pythonpath,
env=subprocess_env,
**subprocess_kwargs)
return Job(command=cmd, process=process)
def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
'--disable-pip-version-check',
# Don't read pip configuration files like `~/.config/pip/pip.conf`.
'--isolated',
# If we want to warn about a version of python we support, we should do it, not pip.
'--no-python-version-warning'
]
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
pex_verbosity = ENV.PEX_VERBOSE
pip_verbosity = pex_verbosity // 3
if pip_verbosity > 0:
pip_args.append('-{}'.format('v' * pip_verbosity))
else:
pip_args.append('-q')
if cache:
pip_args.extend(['--cache-dir', cache])
else:
pip_args.append('--no-cache-dir')
command = pip_args + args
with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(pex_verbosity)) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop('PYTHONPATH', None)
if pythonpath:
TRACER.log('Scrubbed PYTHONPATH={} from the pip PEX environment.'.format(pythonpath), V=3)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(
command=pip.cmdline(command),
process=pip.run(
args=command,
env=env,
blocking=False
)
)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode('utf-8').strip()
if not identity:
raise cls.IdentificationError(
'Could not establish identity of %s' % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode('utf-8'))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed('GCing interpreter cache from prior OS version'):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
cache_dir = os.path.join(os_cache_dir, interpreter_hash)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, 'rb') as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(['pex'])
cmd, env = cls._create_isolated_cmd(binary,
args=[
'-c',
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
],
pythonpath=pythonpath)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of %s" % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
path_id = binary.replace(os.sep, ".").lstrip(".")
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_pip_isolated(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Job
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to
# prompt and we'd also like to know about this since it should never occur. We leverage
# the pip global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup,
# (a)bort.
"--exists-action",
"a",
]
if not package_index_configuration or package_index_configuration.isolated:
# Don't read PIP_ environment variables or pip configuration files like
# `~/.config/pip/pip.conf`.
pip_args.append("--isolated")
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
# of 3.
pip_verbosity = ENV.PEX_VERBOSE // 3
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + list(args)
# N.B.: Package index options in Pep always have the same option names, but they are
# registered as subcommand-specific, so we must append them here _after_ the pip subcommand
# specified in `args`.
if package_index_configuration:
command.extend(package_index_configuration.args)
env = package_index_configuration.env if package_index_configuration else {}
with ENV.strip().patch(
PEX_ROOT=cache or ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE), **env
) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".format(pythonpath), V=3
)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(
command=pip.cmdline(command), process=pip.run(args=command, env=env, blocking=False)
)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout, check_binary=False):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of {}.".format(binary))
interpreter = cls(PythonIdentity.decode(identity))
# We should not need to check this since binary == interpreter.binary should always be
# true, but historically this could be untrue as noted in `PythonIdentity.get`.
if check_binary and not os.path.exists(interpreter.binary):
raise cls.InterpreterNotFound(
"Cached interpreter for {} reports a binary of {}, which could not be found"
.format(binary, interpreter.binary))
return interpreter
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
#
# N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
# length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
# name length limits.
path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(
create_interpreter(fp.read(), check_binary=True))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(binary=binary,
cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def spawn_download_distributions(
self,
download_dir, # type: str
requirements=None, # type: Optional[Iterable[str]]
requirement_files=None, # type: Optional[Iterable[str]]
constraint_files=None, # type: Optional[Iterable[str]]
allow_prereleases=False, # type: bool
transitive=True, # type: bool
target=None, # type: Optional[DistributionTarget]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
build=True, # type: bool
use_wheel=True, # type: bool
):
# type: (...) -> Job
target = target or DistributionTarget.current()
platform, manylinux = target.get_platform()
if not use_wheel:
if not build:
raise ValueError(
"Cannot both ignore wheels (use_wheel=False) and refrain from building "
"distributions (build=False).")
elif target.is_foreign:
raise ValueError(
"Cannot ignore wheels (use_wheel=False) when resolving for a foreign "
"platform: {}".format(platform))
download_cmd = ["download", "--dest", download_dir]
if target.is_foreign:
# We're either resolving for a different host / platform or a different interpreter for
# the current platform that we have no access to; so we need to let pip know and not
# otherwise pickup platform info from the interpreter we execute pip with.
download_cmd.extend(
self._iter_platform_args(
platform=platform.platform,
impl=platform.impl,
version=platform.version,
abi=platform.abi,
manylinux=manylinux,
))
if target.is_foreign or not build:
download_cmd.extend(["--only-binary", ":all:"])
if not use_wheel:
download_cmd.extend(["--no-binary", ":all:"])
if allow_prereleases:
download_cmd.append("--pre")
if not transitive:
download_cmd.append("--no-deps")
if requirement_files:
for requirement_file in requirement_files:
download_cmd.extend(["--requirement", requirement_file])
if constraint_files:
for constraint_file in constraint_files:
download_cmd.extend(["--constraint", constraint_file])
if requirements:
download_cmd.extend(requirements)
# The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed
# with other information we want to suppress. We jump though some hoops here to get at that
# information and surface it on stderr. See: https://github.com/pypa/pip/issues/9420.
log = None
if (self._calculate_resolver_version(
package_index_configuration=package_index_configuration) ==
ResolverVersion.PIP_2020):
log = os.path.join(safe_mkdtemp(), "pip.log")
download_cmd = ["--log", log] + download_cmd
command, process = self._spawn_pip_isolated(
download_cmd,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=target.get_interpreter(),
)
return self._Issue9420Job(command, process, log) if log else Job(
command, process)