def run(self):
if self._installed is not None:
return self._installed
with TRACER.timed('Installing %s' % self._install_tmp, V=2):
env = self._interpreter.sanitized_environment()
mixins = OrderedSet(['setuptools'] + self.mixins)
env['PYTHONPATH'] = os.pathsep.join(third_party.expose(mixins))
env['__PEX_UNVENDORED__'] = '1'
command = [self._interpreter.binary, '-s', '-'] + self.setup_command()
try:
Executor.execute(command,
env=env,
cwd=self._source_dir,
stdin_payload=self.setup_py_wrapper.encode('ascii'))
self._installed = True
except Executor.NonZeroExit as e:
self._installed = False
name = os.path.basename(self._source_dir)
print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr)
print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr)
return self._installed
return self._installed
def create(cls, path=None):
"""Creates a pip tool with PEX isolation at path.
:param str path: The path to build the pip tool pex at; a temporary directory by default.
"""
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=path)
pythonpath = third_party.expose(['pip', 'setuptools', 'wheel'])
isolated_pip_environment = third_party.pkg_resources.Environment(
search_path=pythonpath)
for dist_name in isolated_pip_environment:
for dist in isolated_pip_environment[dist_name]:
isolated_pip_builder.add_dist_location(dist=dist.location)
with open(os.path.join(isolated_pip_builder.path(), 'run_pip.py'),
'w') as fp:
fp.write(
dedent("""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to
# perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""))
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(isolated_pip_builder.path())
def create(
cls,
path, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to assemble the pip tool at.
:param interpreter: The interpreter to run Pip with. The current interpreter by default.
:return: The path of a PEX that can be used to execute Pip in isolation.
"""
pip_interpreter = interpreter or PythonInterpreter.get()
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if not chroot.is_finalized:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
isolated_pip_builder.info.venv = True
for dist_location in third_party.expose(
["pip", "setuptools", "wheel"]):
isolated_pip_builder.add_dist_location(dist=dist_location)
isolated_pip_builder.set_script("pip")
isolated_pip_builder.freeze()
pex_info = PexInfo.from_pex(pip_pex_path)
pex_info.add_interpreter_constraint(
str(pip_interpreter.identity.requirement))
return cls(
ensure_venv(
PEX(pip_pex_path,
interpreter=pip_interpreter,
pex_info=pex_info)))
def _from_binary_external(cls, binary):
pythonpath = third_party.expose(['pex'])
_, stdout, _ = cls._execute(binary,
args=[
'-c',
dedent("""\
import sys
from pex.interpreter import PythonIdentity
sys.stdout.write(PythonIdentity.get().encode())
""")
],
pythonpath=pythonpath)
identity = stdout.strip()
if not identity:
raise cls.IdentificationError(
'Could not establish identity of %s' % binary)
return cls(binary, PythonIdentity.decode(identity))
def spawn_python_job(args,
env=None,
interpreter=None,
expose=None,
pythonpath=None,
**subprocess_kwargs):
"""Spawns a python job.
:param args: The arguments to pass to the python interpreter.
:type args: list of str
:param env: The environment to spawn the python interpreter process in. Defaults to the ambient
environment.
:type env: dict of (str, str)
:param interpreter: The interpreter to use to spawn the python job. Defaults to the current
interpreter.
:type interpreter: :class:`PythonInterpreter`
:param expose: The names of any vendored distributions to expose to the spawned python process.
These will be appended to `pythonpath` if passed.
:type expose: list of str
:param pythonpath: The PYTHONPATH to expose to the spawned python process. These will be
pre-pended to the `expose` path if passed.
:type pythonpath: list of str
:param subprocess_kwargs: Any additional :class:`subprocess.Popen` kwargs to pass through.
:returns: A job handle to the spawned python process.
:rtype: :class:`Job`
"""
pythonpath = list(pythonpath or ())
if expose:
subprocess_env = (env or os.environ).copy()
# In order to expose vendored distributions with their un-vendored import paths in-tact, we
# need to set `__PEX_UNVENDORED__`. See: vendor.__main__.ImportRewriter._modify_import.
subprocess_env['__PEX_UNVENDORED__'] = '1'
pythonpath.extend(third_party.expose(expose))
else:
subprocess_env = env
interpreter = interpreter or PythonInterpreter.get()
cmd, process = interpreter.open_process(args=args,
pythonpath=pythonpath,
env=subprocess_env,
**subprocess_kwargs)
return Job(command=cmd, process=process)
def create(cls, path):
# type: (str) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to build the pip tool pex at.
"""
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if chroot is not None:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot)
pythonpath = third_party.expose(["pip", "setuptools", "wheel"])
isolated_pip_environment = third_party.pkg_resources.Environment(
search_path=pythonpath
)
for dist_name in isolated_pip_environment:
for dist in isolated_pip_environment[dist_name]:
isolated_pip_builder.add_dist_location(dist=dist.location)
with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp:
fp.write(
dedent(
"""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to
# perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""
)
)
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(pip_pex_path)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode('utf-8').strip()
if not identity:
raise cls.IdentificationError(
'Could not establish identity of %s' % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode('utf-8'))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed('GCing interpreter cache from prior OS version'):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
cache_dir = os.path.join(os_cache_dir, interpreter_hash)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, 'rb') as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(['pex'])
cmd, env = cls._create_isolated_cmd(binary,
args=[
'-c',
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
],
pythonpath=pythonpath)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of %s" % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
path_id = binary.replace(os.sep, ".").lstrip(".")
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout, check_binary=False):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of {}.".format(binary))
interpreter = cls(PythonIdentity.decode(identity))
# We should not need to check this since binary == interpreter.binary should always be
# true, but historically this could be untrue as noted in `PythonIdentity.get`.
if check_binary and not os.path.exists(interpreter.binary):
raise cls.InterpreterNotFound(
"Cached interpreter for {} reports a binary of {}, which could not be found"
.format(binary, interpreter.binary))
return interpreter
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
#
# N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
# length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
# name length limits.
path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(
create_interpreter(fp.read(), check_binary=True))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(binary=binary,
cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
