def _spawn_from_binary(cls, binary):
normalized_binary = cls._normalize_path(binary)
# N.B.: The CACHE is written as the last step in PythonInterpreter instance initialization.
cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get(
normalized_binary)
if cached_interpreter is not None:
return SpawnedJob.completed(cached_interpreter)
if normalized_binary == cls._normalize_path(sys.executable):
current_interpreter = cls(PythonIdentity.get())
return SpawnedJob.completed(current_interpreter)
return cls._spawn_from_binary_external(normalized_binary)
def _spawn_from_binary(cls, binary):
# type: (str) -> SpawnedJob[PythonInterpreter]
canonicalized_binary = cls.canonicalize_path(binary)
if not os.path.exists(canonicalized_binary):
raise cls.InterpreterNotFound(canonicalized_binary)
# N.B.: The cache is written as the last step in PythonInterpreter instance initialization.
cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get(
canonicalized_binary)
if cached_interpreter is not None:
return SpawnedJob.completed(cached_interpreter)
if canonicalized_binary == cls.canonicalize_path(sys.executable):
current_interpreter = cls(PythonIdentity.get())
return SpawnedJob.completed(current_interpreter)
return cls._spawn_from_binary_external(canonicalized_binary)
def spawn_calculation(self):
search_path = [dist.location for dist in self.distributions]
program = dedent("""
import json
import sys
from collections import defaultdict
from pkg_resources import Environment
env = Environment(search_path={search_path!r})
dependency_requirements = []
for key in env:
for dist in env[key]:
dependency_requirements.extend(str(req) for req in dist.requires())
json.dump(dependency_requirements, sys.stdout)
""".format(search_path=search_path))
job = spawn_python_job(
args=['-c', program],
stdout=subprocess.PIPE,
interpreter=self.target.get_interpreter(),
expose=['setuptools']
)
return SpawnedJob.stdout(job=job, result_func=self._markers_by_requirement)
def _spawn_wheel_build(self, built_wheels_dir, build_request):
build_result = build_request.result(built_wheels_dir)
build_job = get_pip().spawn_build_wheels(
distributions=[build_request.source_path],
wheel_dir=build_result.build_dir,
cache=self._cache,
interpreter=build_request.target.get_interpreter())
return SpawnedJob.wait(job=build_job, result=build_result)
def _spawn_install(self, installed_wheels_dir, install_request):
install_result = install_request.result(installed_wheels_dir)
install_job = get_pip().spawn_install_wheel(
wheel=install_request.wheel_path,
install_dir=install_result.build_chroot,
compile=self._compile,
cache=self._cache,
target=install_request.target)
return SpawnedJob.wait(job=install_job, result=install_result)
def _spawn_wheel_build(self, built_wheels_dir, build_request):
build_result = build_request.result(built_wheels_dir)
build_job = get_pip().spawn_build_wheels(
distributions=[build_request.source_path],
wheel_dir=build_result.build_dir,
cache=self._cache,
indexes=self._indexes,
find_links=self._find_links,
network_configuration=self._network_configuration,
interpreter=build_request.target.get_interpreter())
return SpawnedJob.wait(job=build_job, result=build_result)
def spawn_extract(distribution):
# type: (Distribution) -> SpawnedJob[Text]
job = spawn_python_job(
args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location],
interpreter=pex.interpreter,
expose=["wheel"],
stdout=subprocess.PIPE,
)
return SpawnedJob.stdout(
job, result_func=lambda out: "{}: {}".format(distribution, out.decode())
)
def _spawn_resolve(self, resolved_dists_dir, target):
download_dir = os.path.join(resolved_dists_dir, target.id)
download_job = get_pip().spawn_download_distributions(
download_dir=download_dir,
requirements=self._requirements,
requirement_files=self._requirement_files,
constraint_files=self._constraint_files,
allow_prereleases=self._allow_prereleases,
transitive=self._transitive,
target=target,
indexes=self._indexes,
find_links=self._find_links,
cache=self._cache,
build=self._build,
manylinux=self._manylinux,
use_wheel=self._use_wheel)
return SpawnedJob.wait(job=download_job,
result=ResolveResult(target, download_dir))
def _spawn_download(self, resolved_dists_dir, target):
download_dir = os.path.join(resolved_dists_dir, target.id)
download_job = get_pip().spawn_download_distributions(
download_dir=download_dir,
requirements=self.requirements,
requirement_files=self.requirement_files,
constraint_files=self.constraint_files,
allow_prereleases=self.allow_prereleases,
transitive=self.transitive,
target=target,
indexes=self.indexes,
find_links=self.find_links,
network_configuration=self.network_configuration,
cache=self.cache,
build=self.build,
manylinux=self.manylinux,
use_wheel=self.use_wheel)
return SpawnedJob.wait(job=download_job,
result=DownloadResult(target, download_dir))
def _calculate_tags(
self,
manylinux=None, # type: Optional[str]
):
# type: (...) -> Iterator[tags.Tag]
from pex.jobs import SpawnedJob
from pex.pip import get_pip
def parse_tags(output):
# type: (bytes) -> Iterator[tags.Tag]
count = None # type: Optional[int]
try:
for line in output.decode("utf-8").splitlines():
if count is None:
match = re.match(
r"^Compatible tags: (?P<count>\d+)\s+", line)
if match:
count = int(match.group("count"))
continue
count -= 1
if count < 0:
raise AssertionError(
"Expected {} tags but got more.".format(count))
for tag in tags.parse_tag(line.strip()):
yield tag
finally:
if count != 0:
raise AssertionError(
"Finished with count {}.".format(count))
job = SpawnedJob.stdout(
job=get_pip().spawn_debug(
platform=self.platform,
impl=self.impl,
version=self.version,
abi=self.abi,
manylinux=manylinux,
),
result_func=parse_tags,
)
return job.await_result()
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode('utf-8').strip()
if not identity:
raise cls.IdentificationError(
'Could not establish identity of %s' % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode('utf-8'))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed('GCing interpreter cache from prior OS version'):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
cache_dir = os.path.join(os_cache_dir, interpreter_hash)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, 'rb') as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(['pex'])
cmd, env = cls._create_isolated_cmd(binary,
args=[
'-c',
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
],
pythonpath=pythonpath)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of %s" % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
path_id = binary.replace(os.sep, ".").lstrip(".")
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout, check_binary=False):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of {}.".format(binary))
interpreter = cls(PythonIdentity.decode(identity))
# We should not need to check this since binary == interpreter.binary should always be
# true, but historically this could be untrue as noted in `PythonIdentity.get`.
if check_binary and not os.path.exists(interpreter.binary):
raise cls.InterpreterNotFound(
"Cached interpreter for {} reports a binary of {}, which could not be found"
.format(binary, interpreter.binary))
return interpreter
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
#
# N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
# length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
# name length limits.
path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(
create_interpreter(fp.read(), check_binary=True))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(binary=binary,
cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
