def _spawn_from_binary(cls, binary): normalized_binary = cls._normalize_path(binary) # N.B.: The CACHE is written as the last step in PythonInterpreter instance initialization. cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get( normalized_binary) if cached_interpreter is not None: return SpawnedJob.completed(cached_interpreter) if normalized_binary == cls._normalize_path(sys.executable): current_interpreter = cls(PythonIdentity.get()) return SpawnedJob.completed(current_interpreter) return cls._spawn_from_binary_external(normalized_binary)
def _spawn_from_binary(cls, binary): # type: (str) -> SpawnedJob[PythonInterpreter] canonicalized_binary = cls.canonicalize_path(binary) if not os.path.exists(canonicalized_binary): raise cls.InterpreterNotFound(canonicalized_binary) # N.B.: The cache is written as the last step in PythonInterpreter instance initialization. cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get( canonicalized_binary) if cached_interpreter is not None: return SpawnedJob.completed(cached_interpreter) if canonicalized_binary == cls.canonicalize_path(sys.executable): current_interpreter = cls(PythonIdentity.get()) return SpawnedJob.completed(current_interpreter) return cls._spawn_from_binary_external(canonicalized_binary)
def spawn_calculation(self): search_path = [dist.location for dist in self.distributions] program = dedent(""" import json import sys from collections import defaultdict from pkg_resources import Environment env = Environment(search_path={search_path!r}) dependency_requirements = [] for key in env: for dist in env[key]: dependency_requirements.extend(str(req) for req in dist.requires()) json.dump(dependency_requirements, sys.stdout) """.format(search_path=search_path)) job = spawn_python_job( args=['-c', program], stdout=subprocess.PIPE, interpreter=self.target.get_interpreter(), expose=['setuptools'] ) return SpawnedJob.stdout(job=job, result_func=self._markers_by_requirement)
def _spawn_wheel_build(self, built_wheels_dir, build_request): build_result = build_request.result(built_wheels_dir) build_job = get_pip().spawn_build_wheels( distributions=[build_request.source_path], wheel_dir=build_result.build_dir, cache=self._cache, interpreter=build_request.target.get_interpreter()) return SpawnedJob.wait(job=build_job, result=build_result)
def _spawn_install(self, installed_wheels_dir, install_request): install_result = install_request.result(installed_wheels_dir) install_job = get_pip().spawn_install_wheel( wheel=install_request.wheel_path, install_dir=install_result.build_chroot, compile=self._compile, cache=self._cache, target=install_request.target) return SpawnedJob.wait(job=install_job, result=install_result)
def _spawn_wheel_build(self, built_wheels_dir, build_request): build_result = build_request.result(built_wheels_dir) build_job = get_pip().spawn_build_wheels( distributions=[build_request.source_path], wheel_dir=build_result.build_dir, cache=self._cache, indexes=self._indexes, find_links=self._find_links, network_configuration=self._network_configuration, interpreter=build_request.target.get_interpreter()) return SpawnedJob.wait(job=build_job, result=build_result)
def spawn_extract(distribution): # type: (Distribution) -> SpawnedJob[Text] job = spawn_python_job( args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location], interpreter=pex.interpreter, expose=["wheel"], stdout=subprocess.PIPE, ) return SpawnedJob.stdout( job, result_func=lambda out: "{}: {}".format(distribution, out.decode()) )
def _spawn_resolve(self, resolved_dists_dir, target): download_dir = os.path.join(resolved_dists_dir, target.id) download_job = get_pip().spawn_download_distributions( download_dir=download_dir, requirements=self._requirements, requirement_files=self._requirement_files, constraint_files=self._constraint_files, allow_prereleases=self._allow_prereleases, transitive=self._transitive, target=target, indexes=self._indexes, find_links=self._find_links, cache=self._cache, build=self._build, manylinux=self._manylinux, use_wheel=self._use_wheel) return SpawnedJob.wait(job=download_job, result=ResolveResult(target, download_dir))
def _spawn_download(self, resolved_dists_dir, target): download_dir = os.path.join(resolved_dists_dir, target.id) download_job = get_pip().spawn_download_distributions( download_dir=download_dir, requirements=self.requirements, requirement_files=self.requirement_files, constraint_files=self.constraint_files, allow_prereleases=self.allow_prereleases, transitive=self.transitive, target=target, indexes=self.indexes, find_links=self.find_links, network_configuration=self.network_configuration, cache=self.cache, build=self.build, manylinux=self.manylinux, use_wheel=self.use_wheel) return SpawnedJob.wait(job=download_job, result=DownloadResult(target, download_dir))
def _calculate_tags( self, manylinux=None, # type: Optional[str] ): # type: (...) -> Iterator[tags.Tag] from pex.jobs import SpawnedJob from pex.pip import get_pip def parse_tags(output): # type: (bytes) -> Iterator[tags.Tag] count = None # type: Optional[int] try: for line in output.decode("utf-8").splitlines(): if count is None: match = re.match( r"^Compatible tags: (?P<count>\d+)\s+", line) if match: count = int(match.group("count")) continue count -= 1 if count < 0: raise AssertionError( "Expected {} tags but got more.".format(count)) for tag in tags.parse_tag(line.strip()): yield tag finally: if count != 0: raise AssertionError( "Finished with count {}.".format(count)) job = SpawnedJob.stdout( job=get_pip().spawn_debug( platform=self.platform, impl=self.impl, version=self.version, abi=self.abi, manylinux=manylinux, ), result_func=parse_tags, ) return job.await_result()
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode('utf-8').strip() if not identity: raise cls.IdentificationError( 'Could not establish identity of %s' % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode('utf-8')) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters') os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed('GCing interpreter cache from prior OS version'): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) cache_dir = os.path.join(os_cache_dir, interpreter_hash) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, 'rb') as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(['pex']) cmd, env = cls._create_isolated_cmd(binary, args=[ '-c', dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)) ], pythonpath=pythonpath) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of %s" % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. path_id = binary.replace(os.sep, ".").lstrip(".") cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout, check_binary=False): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of {}.".format(binary)) interpreter = cls(PythonIdentity.decode(identity)) # We should not need to check this since binary == interpreter.binary should always be # true, but historically this could be untrue as noted in `PythonIdentity.get`. if check_binary and not os.path.exists(interpreter.binary): raise cls.InterpreterNotFound( "Cached interpreter for {} reports a binary of {}, which could not be found" .format(binary, interpreter.binary)) return interpreter # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. # # N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed # length hash of the interpreter binary path to ensure uniqueness and not run afoul of file # name length limits. path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest() cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed( create_interpreter(fp.read(), check_binary=True)) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get(binary={binary!r}).encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(binary=binary, cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)