Beispiel #1
0
    def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
        pip_args = [
            '--disable-pip-version-check', '--isolated', '--exists-action', 'i'
        ]

        # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
        pex_verbosity = ENV.PEX_VERBOSE
        pip_verbosity = pex_verbosity // 3
        if pip_verbosity > 0:
            pip_args.append('-{}'.format('v' * pip_verbosity))
        else:
            pip_args.append('-q')

        if cache:
            pip_args.extend(['--cache-dir', cache])
        else:
            pip_args.append('--no-cache-dir')

        command = pip_args + args
        with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT,
                               PEX_VERBOSE=str(pex_verbosity)) as env:
            from pex.pex import PEX
            pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
            return Job(command=pip.cmdline(command),
                       process=pip.run(args=command, env=env, blocking=False))
Beispiel #2
0
    def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
        pip_args = [
            # We vendor the version of pip we want so pip should never check for updates.
            "--disable-pip-version-check",
            # Don't read pip configuration files like `~/.config/pip/pip.conf`.
            "--isolated",
            # If we want to warn about a version of python we support, we should do it, not pip.
            "--no-python-version-warning",
            # If pip encounters a duplicate file path during its operations we don't want it to prompt
            # and we'd also like to know about this since it should never occur. We leverage the pip
            # global option:
            # --exists-action <action>
            #   Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.
            "--exists-action",
            "a",
        ]

        # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
        pex_verbosity = ENV.PEX_VERBOSE
        pip_verbosity = pex_verbosity // 3
        if pip_verbosity > 0:
            pip_args.append("-{}".format("v" * pip_verbosity))
        else:
            pip_args.append("-q")

        if cache:
            pip_args.extend(["--cache-dir", cache])
        else:
            pip_args.append("--no-cache-dir")

        command = pip_args + args
        with ENV.strip().patch(PEX_ROOT=cache or ENV.PEX_ROOT,
                               PEX_VERBOSE=str(pex_verbosity)) as env:
            # Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
            # bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
            pythonpath = env.pop("PYTHONPATH", None)
            if pythonpath:
                TRACER.log(
                    "Scrubbed PYTHONPATH={} from the pip PEX environment.".
                    format(pythonpath),
                    V=3)

            from pex.pex import PEX

            pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
            return Job(command=pip.cmdline(command),
                       process=pip.run(args=command, env=env, blocking=False))
Beispiel #3
0
 def _spawn_pip_isolated_job(
         self,
         args,  # type: Iterable[str]
         package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
         cache=None,  # type: Optional[str]
         interpreter=None,  # type: Optional[PythonInterpreter]
         pip_verbosity=0,  # type: int
         **popen_kwargs  # type: Any
 ):
     # type: (...) -> Job
     command, process = self._spawn_pip_isolated(
         args,
         package_index_configuration=package_index_configuration,
         cache=cache,
         interpreter=interpreter,
         pip_verbosity=pip_verbosity,
         **popen_kwargs)
     return Job(command=command, process=process)
Beispiel #4
0
def spawn_python_job(args,
                     env=None,
                     interpreter=None,
                     expose=None,
                     pythonpath=None,
                     **subprocess_kwargs):
    """Spawns a python job.

  :param args: The arguments to pass to the python interpreter.
  :type args: list of str
  :param env: The environment to spawn the python interpreter process in. Defaults to the ambient
              environment.
  :type env: dict of (str, str)
  :param interpreter: The interpreter to use to spawn the python job. Defaults to the current
                      interpreter.
  :type interpreter: :class:`PythonInterpreter`
  :param expose: The names of any vendored distributions to expose to the spawned python process.
                 These will be appended to `pythonpath` if passed.
  :type expose: list of str
  :param pythonpath: The PYTHONPATH to expose to the spawned python process. These will be
                     pre-pended to the `expose` path if passed.
  :type pythonpath: list of str
  :param subprocess_kwargs: Any additional :class:`subprocess.Popen` kwargs to pass through.
  :returns: A job handle to the spawned python process.
  :rtype: :class:`Job`
  """
    pythonpath = list(pythonpath or ())
    if expose:
        subprocess_env = (env or os.environ).copy()
        # In order to expose vendored distributions with their un-vendored import paths in-tact, we
        # need to set `__PEX_UNVENDORED__`. See: vendor.__main__.ImportRewriter._modify_import.
        subprocess_env['__PEX_UNVENDORED__'] = '1'

        pythonpath.extend(third_party.expose(expose))
    else:
        subprocess_env = env

    interpreter = interpreter or PythonInterpreter.get()
    cmd, process = interpreter.open_process(args=args,
                                            pythonpath=pythonpath,
                                            env=subprocess_env,
                                            **subprocess_kwargs)
    return Job(command=cmd, process=process)
Beispiel #5
0
  def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
    pip_args = [
      # We vendor the version of pip we want so pip should never check for updates.
      '--disable-pip-version-check',

      # Don't read pip configuration files like `~/.config/pip/pip.conf`.
      '--isolated',

      # If we want to warn about a version of python we support, we should do it, not pip.
      '--no-python-version-warning'
    ]

    # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
    pex_verbosity = ENV.PEX_VERBOSE
    pip_verbosity = pex_verbosity // 3
    if pip_verbosity > 0:
      pip_args.append('-{}'.format('v' * pip_verbosity))
    else:
      pip_args.append('-q')

    if cache:
      pip_args.extend(['--cache-dir', cache])
    else:
      pip_args.append('--no-cache-dir')

    command = pip_args + args
    with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(pex_verbosity)) as env:
      # Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
      # bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
      pythonpath = env.pop('PYTHONPATH', None)
      if pythonpath:
        TRACER.log('Scrubbed PYTHONPATH={} from the pip PEX environment.'.format(pythonpath), V=3)

      from pex.pex import PEX
      pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
      return Job(
        command=pip.cmdline(command),
        process=pip.run(
          args=command,
          env=env,
          blocking=False
        )
      )
Beispiel #6
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout):
            identity = stdout.decode('utf-8').strip()
            if not identity:
                raise cls.IdentificationError(
                    'Could not establish identity of %s' % binary)
            return cls(PythonIdentity.decode(identity))

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode('utf-8'))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed('GCing interpreter cache from prior OS version'):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)
        cache_dir = os.path.join(os_cache_dir, interpreter_hash)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, 'rb') as fp:
                    return SpawnedJob.completed(create_interpreter(fp.read()))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(['pex'])
            cmd, env = cls._create_isolated_cmd(binary,
                                                args=[
                                                    '-c',
                                                    dedent("""\
          import os
          import sys

          from pex.common import atomic_directory, safe_open
          from pex.interpreter import PythonIdentity


          encoded_identity = PythonIdentity.get().encode()
          sys.stdout.write(encoded_identity)
          with atomic_directory({cache_dir!r}) as cache_dir:
            if cache_dir:
              with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                fp.write(encoded_identity)
          """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
                                                ],
                                                pythonpath=pythonpath)
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)
Beispiel #7
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout):
            identity = stdout.decode("utf-8").strip()
            if not identity:
                raise cls.IdentificationError(
                    "Could not establish identity of %s" % binary)
            return cls(PythonIdentity.decode(identity))

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode("utf-8"))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed("GCing interpreter cache from prior OS version"):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)

        # Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
        # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
        # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
        # would otherwise be unstable.
        #
        # See cls._REGEXEN for a related affordance.
        path_id = binary.replace(os.sep, ".").lstrip(".")

        cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, "rb") as fp:
                    return SpawnedJob.completed(create_interpreter(fp.read()))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(["pex"])
            cmd, env = cls._create_isolated_cmd(
                binary,
                args=[
                    "-c",
                    dedent("""\
                        import os
                        import sys

                        from pex.common import atomic_directory, safe_open
                        from pex.interpreter import PythonIdentity


                        encoded_identity = PythonIdentity.get().encode()
                        sys.stdout.write(encoded_identity)
                        with atomic_directory({cache_dir!r}) as cache_dir:
                            if cache_dir:
                                with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                                    fp.write(encoded_identity)
                        """.format(cache_dir=cache_dir,
                                   info_file=cls.INTERP_INFO_FILE)),
                ],
                pythonpath=pythonpath,
            )
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)
Beispiel #8
0
    def _spawn_pip_isolated(
        self,
        args,  # type: Iterable[str]
        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
        cache=None,  # type: Optional[str]
        interpreter=None,  # type: Optional[PythonInterpreter]
    ):
        # type: (...) -> Job
        pip_args = [
            # We vendor the version of pip we want so pip should never check for updates.
            "--disable-pip-version-check",
            # If we want to warn about a version of python we support, we should do it, not pip.
            "--no-python-version-warning",
            # If pip encounters a duplicate file path during its operations we don't want it to
            # prompt and we'd also like to know about this since it should never occur. We leverage
            # the pip global option:
            # --exists-action <action>
            #   Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup,
            #   (a)bort.
            "--exists-action",
            "a",
        ]
        if not package_index_configuration or package_index_configuration.isolated:
            # Don't read PIP_ environment variables or pip configuration files like
            # `~/.config/pip/pip.conf`.
            pip_args.append("--isolated")

        # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
        # of 3.
        pip_verbosity = ENV.PEX_VERBOSE // 3
        if pip_verbosity > 0:
            pip_args.append("-{}".format("v" * pip_verbosity))
        else:
            pip_args.append("-q")

        if cache:
            pip_args.extend(["--cache-dir", cache])
        else:
            pip_args.append("--no-cache-dir")

        command = pip_args + list(args)

        # N.B.: Package index options in Pep always have the same option names, but they are
        # registered as subcommand-specific, so we must append them here _after_ the pip subcommand
        # specified in `args`.
        if package_index_configuration:
            command.extend(package_index_configuration.args)

        env = package_index_configuration.env if package_index_configuration else {}
        with ENV.strip().patch(
            PEX_ROOT=cache or ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE), **env
        ) as env:
            # Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
            # bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
            pythonpath = env.pop("PYTHONPATH", None)
            if pythonpath:
                TRACER.log(
                    "Scrubbed PYTHONPATH={} from the pip PEX environment.".format(pythonpath), V=3
                )

            from pex.pex import PEX

            pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
            return Job(
                command=pip.cmdline(command), process=pip.run(args=command, env=env, blocking=False)
            )
Beispiel #9
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout, check_binary=False):
            identity = stdout.decode("utf-8").strip()
            if not identity:
                raise cls.IdentificationError(
                    "Could not establish identity of {}.".format(binary))
            interpreter = cls(PythonIdentity.decode(identity))
            # We should not need to check this since binary == interpreter.binary should always be
            # true, but historically this could be untrue as noted in `PythonIdentity.get`.
            if check_binary and not os.path.exists(interpreter.binary):
                raise cls.InterpreterNotFound(
                    "Cached interpreter for {} reports a binary of {}, which could not be found"
                    .format(binary, interpreter.binary))
            return interpreter

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode("utf-8"))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed("GCing interpreter cache from prior OS version"):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)

        # Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
        # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
        # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
        # would otherwise be unstable.
        #
        # See cls._REGEXEN for a related affordance.
        #
        # N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
        # length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
        # name length limits.
        path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()

        cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, "rb") as fp:
                    return SpawnedJob.completed(
                        create_interpreter(fp.read(), check_binary=True))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(["pex"])
            cmd, env = cls._create_isolated_cmd(
                binary,
                args=[
                    "-c",
                    dedent("""\
                        import os
                        import sys

                        from pex.common import atomic_directory, safe_open
                        from pex.interpreter import PythonIdentity


                        encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
                        sys.stdout.write(encoded_identity)
                        with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
                            if cache_dir:
                                with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                                    fp.write(encoded_identity)
                        """.format(binary=binary,
                                   cache_dir=cache_dir,
                                   info_file=cls.INTERP_INFO_FILE)),
                ],
                pythonpath=pythonpath,
            )
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)
Beispiel #10
0
    def spawn_download_distributions(
            self,
            download_dir,  # type: str
            requirements=None,  # type: Optional[Iterable[str]]
            requirement_files=None,  # type: Optional[Iterable[str]]
            constraint_files=None,  # type: Optional[Iterable[str]]
            allow_prereleases=False,  # type: bool
            transitive=True,  # type: bool
            target=None,  # type: Optional[DistributionTarget]
            package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
            cache=None,  # type: Optional[str]
            build=True,  # type: bool
            use_wheel=True,  # type: bool
    ):
        # type: (...) -> Job
        target = target or DistributionTarget.current()

        platform, manylinux = target.get_platform()
        if not use_wheel:
            if not build:
                raise ValueError(
                    "Cannot both ignore wheels (use_wheel=False) and refrain from building "
                    "distributions (build=False).")
            elif target.is_foreign:
                raise ValueError(
                    "Cannot ignore wheels (use_wheel=False) when resolving for a foreign "
                    "platform: {}".format(platform))

        download_cmd = ["download", "--dest", download_dir]
        if target.is_foreign:
            # We're either resolving for a different host / platform or a different interpreter for
            # the current platform that we have no access to; so we need to let pip know and not
            # otherwise pickup platform info from the interpreter we execute pip with.
            download_cmd.extend(
                self._iter_platform_args(
                    platform=platform.platform,
                    impl=platform.impl,
                    version=platform.version,
                    abi=platform.abi,
                    manylinux=manylinux,
                ))

        if target.is_foreign or not build:
            download_cmd.extend(["--only-binary", ":all:"])

        if not use_wheel:
            download_cmd.extend(["--no-binary", ":all:"])

        if allow_prereleases:
            download_cmd.append("--pre")

        if not transitive:
            download_cmd.append("--no-deps")

        if requirement_files:
            for requirement_file in requirement_files:
                download_cmd.extend(["--requirement", requirement_file])

        if constraint_files:
            for constraint_file in constraint_files:
                download_cmd.extend(["--constraint", constraint_file])

        if requirements:
            download_cmd.extend(requirements)

        # The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed
        # with other information we want to suppress. We jump though some hoops here to get at that
        # information and surface it on stderr. See: https://github.com/pypa/pip/issues/9420.
        log = None
        if (self._calculate_resolver_version(
                package_index_configuration=package_index_configuration) ==
                ResolverVersion.PIP_2020):
            log = os.path.join(safe_mkdtemp(), "pip.log")
            download_cmd = ["--log", log] + download_cmd

        command, process = self._spawn_pip_isolated(
            download_cmd,
            package_index_configuration=package_index_configuration,
            cache=cache,
            interpreter=target.get_interpreter(),
        )
        return self._Issue9420Job(command, process, log) if log else Job(
            command, process)