def run(self): if self._installed is not None: return self._installed with TRACER.timed('Installing %s' % self._install_tmp, V=2): env = self._interpreter.sanitized_environment() mixins = OrderedSet(['setuptools'] + self.mixins) env['PYTHONPATH'] = os.pathsep.join(third_party.expose(mixins)) env['__PEX_UNVENDORED__'] = '1' command = [self._interpreter.binary, '-s', '-'] + self.setup_command() try: Executor.execute(command, env=env, cwd=self._source_dir, stdin_payload=self.setup_py_wrapper.encode('ascii')) self._installed = True except Executor.NonZeroExit as e: self._installed = False name = os.path.basename(self._source_dir) print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr) print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr) return self._installed return self._installed
def create(cls, path=None): """Creates a pip tool with PEX isolation at path. :param str path: The path to build the pip tool pex at; a temporary directory by default. """ from pex.pex_builder import PEXBuilder isolated_pip_builder = PEXBuilder(path=path) pythonpath = third_party.expose(['pip', 'setuptools', 'wheel']) isolated_pip_environment = third_party.pkg_resources.Environment( search_path=pythonpath) for dist_name in isolated_pip_environment: for dist in isolated_pip_environment[dist_name]: isolated_pip_builder.add_dist_location(dist=dist.location) with open(os.path.join(isolated_pip_builder.path(), 'run_pip.py'), 'w') as fp: fp.write( dedent("""\ import os import runpy import sys # Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to # perform. os.environ['__PEX_UNVENDORED__'] = '1' os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) runpy.run_module('pip', run_name='__main__') """)) isolated_pip_builder.set_executable(fp.name) isolated_pip_builder.freeze() return cls(isolated_pip_builder.path())
def create( cls, path, # type: str interpreter=None, # type: Optional[PythonInterpreter] ): # type: (...) -> Pip """Creates a pip tool with PEX isolation at path. :param path: The path to assemble the pip tool at. :param interpreter: The interpreter to run Pip with. The current interpreter by default. :return: The path of a PEX that can be used to execute Pip in isolation. """ pip_interpreter = interpreter or PythonInterpreter.get() pip_pex_path = os.path.join(path, isolated().pex_hash) with atomic_directory(pip_pex_path, exclusive=True) as chroot: if not chroot.is_finalized: from pex.pex_builder import PEXBuilder isolated_pip_builder = PEXBuilder(path=chroot.work_dir) isolated_pip_builder.info.venv = True for dist_location in third_party.expose( ["pip", "setuptools", "wheel"]): isolated_pip_builder.add_dist_location(dist=dist_location) isolated_pip_builder.set_script("pip") isolated_pip_builder.freeze() pex_info = PexInfo.from_pex(pip_pex_path) pex_info.add_interpreter_constraint( str(pip_interpreter.identity.requirement)) return cls( ensure_venv( PEX(pip_pex_path, interpreter=pip_interpreter, pex_info=pex_info)))
def _from_binary_external(cls, binary): pythonpath = third_party.expose(['pex']) _, stdout, _ = cls._execute(binary, args=[ '-c', dedent("""\ import sys from pex.interpreter import PythonIdentity sys.stdout.write(PythonIdentity.get().encode()) """) ], pythonpath=pythonpath) identity = stdout.strip() if not identity: raise cls.IdentificationError( 'Could not establish identity of %s' % binary) return cls(binary, PythonIdentity.decode(identity))
def spawn_python_job(args, env=None, interpreter=None, expose=None, pythonpath=None, **subprocess_kwargs): """Spawns a python job. :param args: The arguments to pass to the python interpreter. :type args: list of str :param env: The environment to spawn the python interpreter process in. Defaults to the ambient environment. :type env: dict of (str, str) :param interpreter: The interpreter to use to spawn the python job. Defaults to the current interpreter. :type interpreter: :class:`PythonInterpreter` :param expose: The names of any vendored distributions to expose to the spawned python process. These will be appended to `pythonpath` if passed. :type expose: list of str :param pythonpath: The PYTHONPATH to expose to the spawned python process. These will be pre-pended to the `expose` path if passed. :type pythonpath: list of str :param subprocess_kwargs: Any additional :class:`subprocess.Popen` kwargs to pass through. :returns: A job handle to the spawned python process. :rtype: :class:`Job` """ pythonpath = list(pythonpath or ()) if expose: subprocess_env = (env or os.environ).copy() # In order to expose vendored distributions with their un-vendored import paths in-tact, we # need to set `__PEX_UNVENDORED__`. See: vendor.__main__.ImportRewriter._modify_import. subprocess_env['__PEX_UNVENDORED__'] = '1' pythonpath.extend(third_party.expose(expose)) else: subprocess_env = env interpreter = interpreter or PythonInterpreter.get() cmd, process = interpreter.open_process(args=args, pythonpath=pythonpath, env=subprocess_env, **subprocess_kwargs) return Job(command=cmd, process=process)
def create(cls, path): # type: (str) -> Pip """Creates a pip tool with PEX isolation at path. :param path: The path to build the pip tool pex at. """ pip_pex_path = os.path.join(path, isolated().pex_hash) with atomic_directory(pip_pex_path, exclusive=True) as chroot: if chroot is not None: from pex.pex_builder import PEXBuilder isolated_pip_builder = PEXBuilder(path=chroot) pythonpath = third_party.expose(["pip", "setuptools", "wheel"]) isolated_pip_environment = third_party.pkg_resources.Environment( search_path=pythonpath ) for dist_name in isolated_pip_environment: for dist in isolated_pip_environment[dist_name]: isolated_pip_builder.add_dist_location(dist=dist.location) with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp: fp.write( dedent( """\ import os import runpy import sys # Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to # perform. os.environ['__PEX_UNVENDORED__'] = '1' os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) runpy.run_module('pip', run_name='__main__') """ ) ) isolated_pip_builder.set_executable(fp.name) isolated_pip_builder.freeze() return cls(pip_pex_path)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode('utf-8').strip() if not identity: raise cls.IdentificationError( 'Could not establish identity of %s' % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode('utf-8')) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters') os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed('GCing interpreter cache from prior OS version'): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) cache_dir = os.path.join(os_cache_dir, interpreter_hash) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, 'rb') as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(['pex']) cmd, env = cls._create_isolated_cmd(binary, args=[ '-c', dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)) ], pythonpath=pythonpath) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of %s" % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. path_id = binary.replace(os.sep, ".").lstrip(".") cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout, check_binary=False): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of {}.".format(binary)) interpreter = cls(PythonIdentity.decode(identity)) # We should not need to check this since binary == interpreter.binary should always be # true, but historically this could be untrue as noted in `PythonIdentity.get`. if check_binary and not os.path.exists(interpreter.binary): raise cls.InterpreterNotFound( "Cached interpreter for {} reports a binary of {}, which could not be found" .format(binary, interpreter.binary)) return interpreter # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. # # N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed # length hash of the interpreter binary path to ensure uniqueness and not run afoul of file # name length limits. path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest() cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed( create_interpreter(fp.read(), check_binary=True)) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get(binary={binary!r}).encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(binary=binary, cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)