示例#1
0
def test_executor_open_process_communicate():
  process = Executor.open_process(['/bin/echo', '-n', 'hello'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
  stdout, stderr = process.communicate()
  assert stdout.decode('utf-8') == 'hello'
  assert stderr.decode('utf-8') == ''
示例#2
0
文件: pex.py 项目: cosmicexplorer/pex
    def run(self, args=(), with_chroot=False, blocking=True, setsid=False, env=None, **kwargs):
        """Run the PythonEnvironment in an interpreter in a subprocess.

        :keyword args: Additional arguments to be passed to the application being invoked by the
          environment.
        :keyword with_chroot: Run with cwd set to the environment's working directory.
        :keyword blocking: If true, return the return code of the subprocess.
          If false, return the Popen object of the invoked subprocess.
        :keyword setsid: If true, run the PEX in a separate operating system session.
        :keyword env: An optional environment dict to use as the PEX subprocess environment. If none is
                      passed, the ambient environment is inherited.
        Remaining keyword arguments are passed directly to subprocess.Popen.
        """
        if env is not None:
            # If explicit env vars are passed, we don't want clean any of these.
            env = env.copy()
        else:
            env = os.environ.copy()
            self._clean_environment(env=env)

        cmdline = self.cmdline(args)
        TRACER.log("PEX.run invoking %s" % " ".join(cmdline))
        process = Executor.open_process(
            cmdline,
            cwd=self._pex if with_chroot else os.getcwd(),
            preexec_fn=os.setsid if setsid else None,
            stdin=kwargs.pop("stdin", None),
            stdout=kwargs.pop("stdout", None),
            stderr=kwargs.pop("stderr", None),
            env=env,
            **kwargs
        )
        return process.wait() if blocking else process
示例#3
0
    def run(self,
            args=(),
            with_chroot=False,
            blocking=True,
            setsid=False,
            **kwargs):
        """Run the PythonEnvironment in an interpreter in a subprocess.

    :keyword args: Additional arguments to be passed to the application being invoked by the
      environment.
    :keyword with_chroot: Run with cwd set to the environment's working directory.
    :keyword blocking: If true, return the return code of the subprocess.
      If false, return the Popen object of the invoked subprocess.
    :keyword setsid: If true, run the PEX in a separate operating system session.

    Remaining keyword arguments are passed directly to subprocess.Popen.
    """
        self.clean_environment()

        cmdline = self.cmdline(args)
        TRACER.log('PEX.run invoking %s' % ' '.join(cmdline))
        process = Executor.open_process(
            cmdline,
            cwd=self._pex if with_chroot else os.getcwd(),
            preexec_fn=os.setsid if setsid else None,
            stdin=kwargs.pop('stdin', None),
            stdout=kwargs.pop('stdout', None),
            stderr=kwargs.pop('stderr', None),
            **kwargs)
        return process.wait() if blocking else process
示例#4
0
    def run(self):
        parser, options_builder = configure_clp()
        options, reqs = parser.parse_args(self.pex_args)

        if options.entry_point or options.script or options.pex_name:
            die('Must not specify entry point, script or output file to --pex-args, given: {}'
                .format(' '.join(self.pex_args)))

        name = self.distribution.get_name()
        version = self.distribution.get_version()

        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))
        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, 'dist')

        console_scripts = self.parse_entry_points()

        pex_specs = []
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            pex_specs.extend(
                (script_name, os.path.join(self.bdist_dir, script_name))
                for script_name in console_scripts)
        else:
            target = os.path.join(self.bdist_dir,
                                  name + '-' + version + '.pex')
            pex_specs.append(
                (name if name in console_scripts else None, target))

        # In order for code to run to here, pex is on the sys.path - make sure to propagate the
        # sys.path so the subprocess can find us.
        env = os.environ.copy()
        env['PYTHONPATH'] = os.pathsep.join(sys.path)

        args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir
                ] + reqs + self.pex_args
        if self.get_log_level() < log.INFO and options.verbosity == 0:
            args.append('-v')

        for script_name, target in pex_specs:
            cmd = args + ['--output-file', target]
            if script_name:
                log.info('Writing %s to %s' % (script_name, target))
                cmd += ['--script', script_name]
            else:
                # The package has no namesake entry point, so build an environment pex.
                log.info('Writing environment pex into %s' % target)

            log.debug('Building pex via: {}'.format(' '.join(cmd)))
            process = Executor.open_process(cmd,
                                            stderr=subprocess.PIPE,
                                            env=env)
            _, stderr = process.communicate()
            result = process.returncode
            if result != 0:
                die(
                    'Failed to create pex via {}:\n{}'.format(
                        ' '.join(cmd), stderr.decode('utf-8')), result)
示例#5
0
def test_executor_open_process_communicate():
    process = Executor.open_process(["/bin/echo", "-n", "hello"],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
    stdout, stderr = process.communicate()
    assert stdout.decode("utf-8") == "hello"
    assert stderr.decode("utf-8") == ""
示例#6
0
 def open_process(self, args=None, pythonpath=None, env=None, **kwargs):
     cmd, env = self._create_isolated_cmd(self.binary,
                                          args=args,
                                          pythonpath=pythonpath,
                                          env=env)
     process = Executor.open_process(cmd, env=env, **kwargs)
     return cmd, process
示例#7
0
def test_executor_open_process_communicate():
  process = Executor.open_process(['/bin/echo', '-n', 'hello'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
  stdout, stderr = process.communicate()
  assert stdout.decode('utf-8') == 'hello'
  assert stderr.decode('utf-8') == ''
示例#8
0
文件: testing.py 项目: jsirois/pex
def run_pex_command(args, env=None):
  """Simulate running pex command for integration testing.

  This is different from run_simple_pex in that it calls the pex command rather
  than running a generated pex.  This is useful for testing end to end runs
  with specific command line arguments or env options.
  """
  cmd = [sys.executable, '-mpex', '-vvvvv'] + list(args)
  process = Executor.open_process(cmd=cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  output, error = process.communicate()
  return IntegResults(output.decode('utf-8'), error.decode('utf-8'), process.returncode)
示例#9
0
文件: bdist_pex.py 项目: jsirois/pex
  def run(self):
    parser, options_builder = configure_clp()
    options, reqs = parser.parse_args(self.pex_args)

    if options.entry_point or options.script or options.pex_name:
      die('Must not specify entry point, script or output file to --pex-args, given: {}'
          .format(' '.join(self.pex_args)))

    name = self.distribution.get_name()
    version = self.distribution.get_version()

    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
      self.distribution.script_name)))
    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    console_scripts = self.parse_entry_points()

    pex_specs = []
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name))
                       for script_name in console_scripts)
    else:
      target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
      pex_specs.append((name if name in console_scripts else None, target))

    # In order for code to run to here, pex is on the sys.path - make sure to propagate the
    # sys.path so the subprocess can find us.
    env = os.environ.copy()
    env['PYTHONPATH'] = os.pathsep.join(sys.path)

    args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args
    if self.get_log_level() < log.INFO and options.verbosity == 0:
      args.append('-v')

    for script_name, target in pex_specs:
      cmd = args + ['--output-file', target]
      if script_name:
        log.info('Writing %s to %s' % (script_name, target))
        cmd += ['--script', script_name]
      else:
        # The package has no namesake entry point, so build an environment pex.
        log.info('Writing environment pex into %s' % target)

      log.debug('Building pex via: {}'.format(' '.join(cmd)))
      process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
      _, stderr = process.communicate()
      result = process.returncode
      if result != 0:
        die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')),
            result)
示例#10
0
 def open_process(
         self,
         args=None,  # type: Optional[Iterable[str]]
         pythonpath=None,  # type: Optional[Iterable[str]]
         env=None,  # type: Optional[Mapping[str, str]]
         **kwargs  # type: Any
 ):
     # type: (...) -> Tuple[Iterable[str], subprocess.Popen]
     cmd, env = self.create_isolated_cmd(args=args,
                                         pythonpath=pythonpath,
                                         env=env)
     process = Executor.open_process(cmd, env=env, **kwargs)
     return cmd, process
示例#11
0
文件: testing.py 项目: tdyas/pex
def run_pex_command(args, env=None, python=None, quiet=False):
  """Simulate running pex command for integration testing.

  This is different from run_simple_pex in that it calls the pex command rather
  than running a generated pex.  This is useful for testing end to end runs
  with specific command line arguments or env options.
  """
  cmd = [python or sys.executable, '-mpex']
  if not quiet:
    cmd.append('-vvvvv')
  cmd.extend(args)
  process = Executor.open_process(cmd=cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  output, error = process.communicate()
  return IntegResults(output.decode('utf-8'), error.decode('utf-8'), process.returncode)
示例#12
0
def test_pex_builder_add_source_relpath_issues_1192(
        tmp_chroot,  # type: str
        copy_mode,  # type: CopyMode.Value
):
    # type: (...) -> None
    pb = PEXBuilder(copy_mode=copy_mode)
    with safe_open("src/main.py", "w") as fp:
        fp.write("import sys; sys.exit(42)")
    pb.add_source("src/main.py", "main.py")
    pb.set_entry_point("main")
    pb.build("test.pex")

    process = Executor.open_process(cmd=[os.path.abspath("test.pex")])
    process.wait()
    assert 42 == process.returncode
示例#13
0
文件: pex.py 项目: jsirois/pex
  def run(self, args=(), with_chroot=False, blocking=True, setsid=False, **kwargs):
    """Run the PythonEnvironment in an interpreter in a subprocess.

    :keyword args: Additional arguments to be passed to the application being invoked by the
      environment.
    :keyword with_chroot: Run with cwd set to the environment's working directory.
    :keyword blocking: If true, return the return code of the subprocess.
      If false, return the Popen object of the invoked subprocess.
    :keyword setsid: If true, run the PEX in a separate operating system session.

    Remaining keyword arguments are passed directly to subprocess.Popen.
    """
    self.clean_environment()

    cmdline = self.cmdline(args)
    TRACER.log('PEX.run invoking %s' % ' '.join(cmdline))
    process = Executor.open_process(cmdline,
                                    cwd=self._pex if with_chroot else os.getcwd(),
                                    preexec_fn=os.setsid if setsid else None,
                                    stdin=kwargs.pop('stdin', None),
                                    stdout=kwargs.pop('stdout', None),
                                    stderr=kwargs.pop('stderr', None),
                                    **kwargs)
    return process.wait() if blocking else process
示例#14
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout):
            identity = stdout.decode("utf-8").strip()
            if not identity:
                raise cls.IdentificationError(
                    "Could not establish identity of %s" % binary)
            return cls(PythonIdentity.decode(identity))

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode("utf-8"))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed("GCing interpreter cache from prior OS version"):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)

        # Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
        # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
        # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
        # would otherwise be unstable.
        #
        # See cls._REGEXEN for a related affordance.
        path_id = binary.replace(os.sep, ".").lstrip(".")

        cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, "rb") as fp:
                    return SpawnedJob.completed(create_interpreter(fp.read()))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(["pex"])
            cmd, env = cls._create_isolated_cmd(
                binary,
                args=[
                    "-c",
                    dedent("""\
                        import os
                        import sys

                        from pex.common import atomic_directory, safe_open
                        from pex.interpreter import PythonIdentity


                        encoded_identity = PythonIdentity.get().encode()
                        sys.stdout.write(encoded_identity)
                        with atomic_directory({cache_dir!r}) as cache_dir:
                            if cache_dir:
                                with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                                    fp.write(encoded_identity)
                        """.format(cache_dir=cache_dir,
                                   info_file=cls.INTERP_INFO_FILE)),
                ],
                pythonpath=pythonpath,
            )
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)
示例#15
0
def test_executor_open_process_wait_return():
    # type: () -> None
    process = Executor.open_process("exit 8", shell=True)
    exit_code = process.wait()
    assert exit_code == 8
示例#16
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout, check_binary=False):
            identity = stdout.decode("utf-8").strip()
            if not identity:
                raise cls.IdentificationError(
                    "Could not establish identity of {}.".format(binary))
            interpreter = cls(PythonIdentity.decode(identity))
            # We should not need to check this since binary == interpreter.binary should always be
            # true, but historically this could be untrue as noted in `PythonIdentity.get`.
            if check_binary and not os.path.exists(interpreter.binary):
                raise cls.InterpreterNotFound(
                    "Cached interpreter for {} reports a binary of {}, which could not be found"
                    .format(binary, interpreter.binary))
            return interpreter

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode("utf-8"))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed("GCing interpreter cache from prior OS version"):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)

        # Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
        # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
        # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
        # would otherwise be unstable.
        #
        # See cls._REGEXEN for a related affordance.
        #
        # N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
        # length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
        # name length limits.
        path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()

        cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, "rb") as fp:
                    return SpawnedJob.completed(
                        create_interpreter(fp.read(), check_binary=True))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(["pex"])
            cmd, env = cls._create_isolated_cmd(
                binary,
                args=[
                    "-c",
                    dedent("""\
                        import os
                        import sys

                        from pex.common import atomic_directory, safe_open
                        from pex.interpreter import PythonIdentity


                        encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
                        sys.stdout.write(encoded_identity)
                        with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
                            if cache_dir:
                                with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                                    fp.write(encoded_identity)
                        """.format(binary=binary,
                                   cache_dir=cache_dir,
                                   info_file=cls.INTERP_INFO_FILE)),
                ],
                pythonpath=pythonpath,
            )
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)
示例#17
0
def test_executor_open_process_wait_return():
  process = Executor.open_process('exit 8', shell=True)
  exit_code = process.wait()
  assert exit_code == 8
示例#18
0
def test_executor_open_process_wait_return():
  process = Executor.open_process('exit 8', shell=True)
  exit_code = process.wait()
  assert exit_code == 8
示例#19
0
    def _spawn_from_binary_external(cls, binary):
        def create_interpreter(stdout):
            identity = stdout.decode('utf-8').strip()
            if not identity:
                raise cls.IdentificationError(
                    'Could not establish identity of %s' % binary)
            return cls(PythonIdentity.decode(identity))

        # Part of the PythonInterpreter data are environment markers that depend on the current OS
        # release. That data can change when the OS is upgraded but (some of) the installed interpreters
        # remain the same. As such, include the OS in the hash structure for cached interpreters.
        os_digest = hashlib.sha1()
        for os_identifier in platform.release(), platform.version():
            os_digest.update(os_identifier.encode('utf-8'))
        os_hash = os_digest.hexdigest()

        interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
        os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
        if os.path.isdir(
                interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
            with TRACER.timed('GCing interpreter cache from prior OS version'):
                safe_rmtree(interpreter_cache_dir)

        interpreter_hash = CacheHelper.hash(binary)
        cache_dir = os.path.join(os_cache_dir, interpreter_hash)
        cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
        if os.path.isfile(cache_file):
            try:
                with open(cache_file, 'rb') as fp:
                    return SpawnedJob.completed(create_interpreter(fp.read()))
            except (IOError, OSError, cls.Error, PythonIdentity.Error):
                safe_rmtree(cache_dir)
                return cls._spawn_from_binary_external(binary)
        else:
            pythonpath = third_party.expose(['pex'])
            cmd, env = cls._create_isolated_cmd(binary,
                                                args=[
                                                    '-c',
                                                    dedent("""\
          import os
          import sys

          from pex.common import atomic_directory, safe_open
          from pex.interpreter import PythonIdentity


          encoded_identity = PythonIdentity.get().encode()
          sys.stdout.write(encoded_identity)
          with atomic_directory({cache_dir!r}) as cache_dir:
            if cache_dir:
              with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
                fp.write(encoded_identity)
          """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
                                                ],
                                                pythonpath=pythonpath)
            process = Executor.open_process(cmd,
                                            env=env,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
            job = Job(command=cmd, process=process)
            return SpawnedJob.stdout(job, result_func=create_interpreter)