Esempio n. 1
0
    def _wrap_coverage(self, runner, *args):
        if not self._vars.PEX_COVERAGE and self._vars.PEX_COVERAGE_FILENAME is None:
            return runner(*args)

        try:
            import coverage
        except ImportError:
            die('Could not bootstrap coverage module, aborting.')

        pex_coverage_filename = self._vars.PEX_COVERAGE_FILENAME
        if pex_coverage_filename is not None:
            cov = coverage.coverage(data_file=pex_coverage_filename)
        else:
            cov = coverage.coverage(data_suffix=True)

        TRACER.log('Starting coverage.')
        cov.start()

        try:
            return runner(*args)
        finally:
            TRACER.log('Stopping coverage')
            cov.stop()

            # TODO(wickman) Post-process coverage to elide $PEX_ROOT and make
            # the report more useful/less noisy.  #89
            if pex_coverage_filename:
                cov.save()
            else:
                cov.report(show_missing=False,
                           ignore_errors=True,
                           file=sys.stdout)
Esempio n. 2
0
    def _execute(self):
        force_interpreter = self._vars.PEX_INTERPRETER

        # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been
        # executed from within a PEX zip file in `--unzip` mode.  We replace `sys.argv[0]` to avoid
        # confusion and allow the user code we hand off to to provide useful messages and fully valid
        # re-execs that always re-directed through the PEX file.
        sys.argv[0] = os.environ.pop('__PEX_EXE__', sys.argv[0])

        self._clean_environment(strip_pex_env=self._pex_info.strip_pex_env)

        if force_interpreter:
            TRACER.log('PEX_INTERPRETER specified, dropping into interpreter')
            return self.execute_interpreter()

        if self._pex_info_overrides.script and self._pex_info_overrides.entry_point:
            die('Cannot specify both script and entry_point for a PEX!')

        if self._pex_info.script and self._pex_info.entry_point:
            die('Cannot specify both script and entry_point for a PEX!')

        if self._pex_info_overrides.script:
            return self.execute_script(self._pex_info_overrides.script)
        elif self._pex_info_overrides.entry_point:
            return self.execute_entry(self._pex_info_overrides.entry_point)
        elif self._pex_info.script:
            return self.execute_script(self._pex_info.script)
        elif self._pex_info.entry_point:
            return self.execute_entry(self._pex_info.entry_point)
        else:
            TRACER.log('No entry point specified, dropping into interpreter')
            return self.execute_interpreter()
Esempio n. 3
0
File: pex.py Progetto: jsirois/pex
  def _execute(self):
    force_interpreter = self._vars.PEX_INTERPRETER

    self.clean_environment()

    if force_interpreter:
      TRACER.log('PEX_INTERPRETER specified, dropping into interpreter')
      return self.execute_interpreter()

    if self._pex_info_overrides.script and self._pex_info_overrides.entry_point:
      die('Cannot specify both script and entry_point for a PEX!')

    if self._pex_info.script and self._pex_info.entry_point:
      die('Cannot specify both script and entry_point for a PEX!')

    if self._pex_info_overrides.script:
      return self.execute_script(self._pex_info_overrides.script)
    elif self._pex_info_overrides.entry_point:
      return self.execute_entry(self._pex_info_overrides.entry_point)
    elif self._pex_info.script:
      return self.execute_script(self._pex_info.script)
    elif self._pex_info.entry_point:
      return self.execute_entry(self._pex_info.entry_point)
    else:
      TRACER.log('No entry point specified, dropping into interpreter')
      return self.execute_interpreter()
Esempio n. 4
0
    def execute_interpreter(self):
        args = sys.argv[1:]
        if args:
            # NB: We take care here to setup sys.argv to match how CPython does it for each case.
            arg = args[0]
            if arg == '-c':
                content = args[1]
                sys.argv = ['-c'] + args[2:]
                self.execute_content('-c <cmd>', content, argv0='-c')
            elif arg == '-m':
                module = args[1]
                sys.argv = args[1:]
                self.execute_module(module)
            else:
                try:
                    if arg == '-':
                        content = sys.stdin.read()
                    else:
                        with open(arg) as fp:
                            content = fp.read()
                except IOError as e:
                    die("Could not open %s in the environment [%s]: %s" %
                        (arg, sys.argv[0], e))
                sys.argv = args
                self.execute_content(arg, content)
        else:
            self.demote_bootstrap()

            import code
            code.interact()
Esempio n. 5
0
def find_compatible_interpreters(pex_python_path=None, compatibility_constraints=None):
  """Find all compatible interpreters on the system within the supplied constraints and use
     PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
  """
  if pex_python_path:
    interpreters = []
    for binary in pex_python_path.split(os.pathsep):
      try:
        interpreters.append(PythonInterpreter.from_binary(binary))
      except Executor.ExecutionError:
        print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary,
          file=sys.stderr)
    if not interpreters:
      die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.')
  else:
    # We may have been invoked with a specific interpreter not on the $PATH, make sure our
    # sys.executable is included as a candidate in this case.
    interpreters = OrderedSet([PythonInterpreter.get()])

    # Add all qualifying interpreters found in $PATH.
    interpreters.update(PythonInterpreter.all())

  return list(
    matched_interpreters(interpreters, compatibility_constraints)
    if compatibility_constraints
    else interpreters
  )
Esempio n. 6
0
 def _get_int(self, variable, default=None):
   try:
     return int(self._environ[variable])
   except ValueError:
     die('Invalid value for %s, must be an integer, got %r' % (variable, self._environ[variable]))
   except KeyError:
     return self._defaulted(default)
Esempio n. 7
0
def _select_path_interpreter(path=None, compatibility_constraints=None):
    compatible_interpreters = find_compatible_interpreters(
        path=path, compatibility_constraints=compatibility_constraints)
    if not compatible_interpreters:
        die('Failed to find compatible interpreter on path {} for constraints: {}'
            .format(path or os.getenv('PATH'), compatibility_constraints))
    return _select_interpreter(compatible_interpreters)
Esempio n. 8
0
 def _get_int(self, variable, default=None):
   try:
     return int(self._environ[variable])
   except ValueError:
     die('Invalid value for %s, must be an integer, got %r' % (variable, self._environ[variable]))
   except KeyError:
     return self._defaulted(default)
Esempio n. 9
0
def resolve_or_die(interpreter, requirement, options):
    resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)

    interpreter = resolve(interpreter, requirement)
    if interpreter is None:
      die('Could not find compatible interpreter that meets requirement %s' % requirement, CANNOT_SETUP_INTERPRETER)
    return interpreter
Esempio n. 10
0
    def run(self):
        parser, options_builder = configure_clp()
        options, reqs = parser.parse_args(self.pex_args)

        if options.entry_point or options.script or options.pex_name:
            die('Must not specify entry point, script or output file to --pex-args, given: {}'
                .format(' '.join(self.pex_args)))

        name = self.distribution.get_name()
        version = self.distribution.get_version()

        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))
        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, 'dist')

        console_scripts = self.parse_entry_points()

        pex_specs = []
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            pex_specs.extend(
                (script_name, os.path.join(self.bdist_dir, script_name))
                for script_name in console_scripts)
        else:
            target = os.path.join(self.bdist_dir,
                                  name + '-' + version + '.pex')
            pex_specs.append(
                (name if name in console_scripts else None, target))

        # In order for code to run to here, pex is on the sys.path - make sure to propagate the
        # sys.path so the subprocess can find us.
        env = os.environ.copy()
        env['PYTHONPATH'] = os.pathsep.join(sys.path)

        args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir
                ] + reqs + self.pex_args
        if self.get_log_level() < log.INFO and options.verbosity == 0:
            args.append('-v')

        for script_name, target in pex_specs:
            cmd = args + ['--output-file', target]
            if script_name:
                log.info('Writing %s to %s' % (script_name, target))
                cmd += ['--script', script_name]
            else:
                # The package has no namesake entry point, so build an environment pex.
                log.info('Writing environment pex into %s' % target)

            log.debug('Building pex via: {}'.format(' '.join(cmd)))
            process = Executor.open_process(cmd,
                                            stderr=subprocess.PIPE,
                                            env=env)
            _, stderr = process.communicate()
            result = process.returncode
            if result != 0:
                die(
                    'Failed to create pex via {}:\n{}'.format(
                        ' '.join(cmd), stderr.decode('utf-8')), result)
Esempio n. 11
0
    def execute(self):
        # type: () -> None
        """Execute the PEX.

        This function makes assumptions that it is the last function called by the interpreter.
        """
        teardown_verbosity = self._vars.PEX_TEARDOWN_VERBOSE

        # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been
        # executed from within a PEX zip file in `--unzip` mode.  We replace `sys.argv[0]` to avoid
        # confusion and allow the user code we hand off to to provide useful messages and fully
        # valid re-execs that always re-directed through the PEX file.
        sys.argv[0] = os.environ.pop("__PEX_EXE__", sys.argv[0])

        try:
            if self._vars.PEX_TOOLS:
                try:
                    from pex.tools import main as tools
                except ImportError as e:
                    die(
                        "The PEX_TOOLS environment variable was set, but this PEX was not built "
                        "with tools (Re-build the PEX file with `pex --include-tools ...`):"
                        " {}".format(e)
                    )

                exit_value = tools.main(pex=self, pex_prog_path=sys.argv[0])
            else:
                self.activate()
                exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
            sys.exit(exit_value)
        except Exception:
            # Allow the current sys.excepthook to handle this app exception before we tear things
            # down in finally, then reraise so that the exit status is reflected correctly.
            sys.excepthook(*sys.exc_info())
            raise
        except SystemExit as se:
            # Print a SystemExit error message, avoiding a traceback in python3.
            # This must happen here, as sys.stderr is about to be torn down
            if not isinstance(se.code, int) and se.code is not None:  # type: ignore[unreachable]
                print(se.code, file=sys.stderr)  # type: ignore[unreachable]
            raise
        finally:
            # squash all exceptions on interpreter teardown -- the primary type here are
            # atexit handlers failing to run because of things such as:
            #   http://stackoverflow.com/questions/2572172/referencing-other-modules-in-atexit
            if not teardown_verbosity:
                sys.stderr.flush()
                sys.stderr = open(os.devnull, "w")
                if PY3:
                    # Python 3 warns about unclosed resources. In this case we intentionally do not
                    # close `/dev/null` since we want all stderr to flow there until the latest
                    # stages of Python interpreter shutdown when the Python runtime will `del` the
                    # open file and thus finally close the underlying file descriptor. As such,
                    # suppress the warning.
                    warnings.filterwarnings(
                        action="ignore",
                        message=r"unclosed file {}".format(re.escape(str(sys.stderr))),
                        category=ResourceWarning,
                    )
                sys.excepthook = lambda *a, **kw: None
Esempio n. 12
0
File: pex.py Progetto: t-cas/pex
def get_interpreter(python_interpreter, interpreter_cache_dir, repos,
                    use_wheel):
    interpreter = None

    if python_interpreter:
        if os.path.exists(python_interpreter):
            interpreter = PythonInterpreter.from_binary(python_interpreter)
        else:
            interpreter = PythonInterpreter.from_env(python_interpreter)
        if interpreter is None:
            die('Failed to find interpreter: %s' % python_interpreter)
    else:
        interpreter = PythonInterpreter.get()

    with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
        resolve = functools.partial(resolve_interpreter, interpreter_cache_dir,
                                    repos)

        # resolve setuptools
        interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)

        # possibly resolve wheel
        if interpreter and use_wheel:
            interpreter = resolve(interpreter, WHEEL_REQUIREMENT)

        return interpreter
Esempio n. 13
0
File: pex.py Progetto: jsirois/pex
  def execute_interpreter(self):
    args = sys.argv[1:]
    if args:
      # NB: We take care here to setup sys.argv to match how CPython does it for each case.
      arg = args[0]
      if arg == '-c':
        content = args[1]
        sys.argv = ['-c'] + args[2:]
        self.execute_content('-c <cmd>', content, argv0='-c')
      elif arg == '-m':
        module = args[1]
        sys.argv = args[1:]
        self.execute_module(module)
      else:
        try:
          if arg == '-':
            content = sys.stdin.read()
          else:
            with open(arg) as fp:
              content = fp.read()
        except IOError as e:
          die("Could not open %s in the environment [%s]: %s" % (arg, sys.argv[0], e))
        sys.argv = args
        self.execute_content(arg, content)
    else:
      self.demote_bootstrap()

      import code
      code.interact()
Esempio n. 14
0
    def _execute(self):
        force_interpreter = self._vars.PEX_INTERPRETER

        self.clean_environment()

        if force_interpreter:
            TRACER.log('PEX_INTERPRETER specified, dropping into interpreter')
            return self.execute_interpreter()

        if self._pex_info_overrides.script and self._pex_info_overrides.entry_point:
            die('Cannot specify both script and entry_point for a PEX!')

        if self._pex_info.script and self._pex_info.entry_point:
            die('Cannot specify both script and entry_point for a PEX!')

        if self._pex_info_overrides.script:
            return self.execute_script(self._pex_info_overrides.script)
        elif self._pex_info_overrides.entry_point:
            return self.execute_entry(self._pex_info_overrides.entry_point)
        elif self._pex_info.script:
            return self.execute_script(self._pex_info.script)
        elif self._pex_info.entry_point:
            return self.execute_entry(self._pex_info.entry_point)
        else:
            TRACER.log('No entry point specified, dropping into interpreter')
            return self.execute_interpreter()
Esempio n. 15
0
File: pex.py Progetto: windie/heron
def interpreter_from_options(options):
    interpreter = None

    if options.python:
        if os.path.exists(options.python):
            interpreter = PythonInterpreter.from_binary(options.python)
        else:
            interpreter = PythonInterpreter.from_env(options.python)
        if interpreter is None:
            die('Failed to find interpreter: %s' % options.python)
    else:
        interpreter = PythonInterpreter.get()

    with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
        resolve = functools.partial(resolve_interpreter,
                                    options.interpreter_cache_dir,
                                    options.repos)

        # resolve setuptools
        interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)

        # possibly resolve wheel
        if interpreter and options.use_wheel:
            interpreter = resolve(interpreter, WHEEL_REQUIREMENT)

        return interpreter
Esempio n. 16
0
def find_compatible_interpreters(pex_python_path, compatibility_constraints):
    """Find all compatible interpreters on the system within the supplied constraints and use
     PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
  """
    if pex_python_path:
        interpreters = []
        for binary in pex_python_path.split(os.pathsep):
            try:
                interpreters.append(PythonInterpreter.from_binary(binary))
            except Executor.ExecutionError:
                print(
                    "Python interpreter %s in PEX_PYTHON_PATH failed to load properly."
                    % binary,
                    file=sys.stderr)
        if not interpreters:
            die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.'
                )
    else:
        if not os.getenv('PATH', ''):
            # no $PATH, use sys.executable
            interpreters = [PythonInterpreter.get()]
        else:
            # get all qualifying interpreters found in $PATH
            interpreters = PythonInterpreter.all()

    return list(
        matched_interpreters(interpreters, compatibility_constraints
                             ) if compatibility_constraints else interpreters)
Esempio n. 17
0
File: pex.py Progetto: jsirois/pex
  def _wrap_coverage(self, runner, *args):
    if not self._vars.PEX_COVERAGE and self._vars.PEX_COVERAGE_FILENAME is None:
      return runner(*args)

    try:
      import coverage
    except ImportError:
      die('Could not bootstrap coverage module, aborting.')

    pex_coverage_filename = self._vars.PEX_COVERAGE_FILENAME
    if pex_coverage_filename is not None:
      cov = coverage.coverage(data_file=pex_coverage_filename)
    else:
      cov = coverage.coverage(data_suffix=True)

    TRACER.log('Starting coverage.')
    cov.start()

    try:
      return runner(*args)
    finally:
      TRACER.log('Stopping coverage')
      cov.stop()

      # TODO(wickman) Post-process coverage to elide $PEX_ROOT and make
      # the report more useful/less noisy.  #89
      if pex_coverage_filename:
        cov.save()
      else:
        cov.report(show_missing=False, ignore_errors=True, file=sys.stdout)
Esempio n. 18
0
    def run(self):
        parser = configure_clp()
        options, reqs = parser.parse_args(self.pex_args)

        if options.entry_point or options.script or options.pex_name:
            die("Must not specify entry point, script or output file to --pex-args, given: {}"
                .format(" ".join(self.pex_args)))

        name = self.distribution.get_name()
        version = self.distribution.get_version()

        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))
        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, "dist")

        console_scripts = self.parse_entry_points()

        pex_specs = []
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            pex_specs.extend(
                (script_name, os.path.join(self.bdist_dir, script_name))
                for script_name in console_scripts)
        else:
            target = os.path.join(self.bdist_dir,
                                  name + "-" + version + ".pex")
            pex_specs.append(
                (name if name in console_scripts else None, target))

        args = ["-m", "pex", package_dir] + reqs + self.pex_args
        if self.get_log_level() < log.INFO and options.verbosity == 0:
            args.append("-v")

        for script_name, target in pex_specs:
            pex_cmd = args + ["--output-file", target]
            if script_name:
                log.info("Writing %s to %s" % (script_name, target))
                pex_cmd += ["--script", script_name]
            else:
                # The package has no namesake entry point, so build an environment pex.
                log.info("Writing environment pex into %s" % target)

            cmd, process = PythonInterpreter.get().open_process(
                args=pex_cmd,
                stderr=subprocess.PIPE,
                # In order for code to run to here, pex is on the sys.path - make sure to propagate the
                # sys.path so the subprocess can find us.
                pythonpath=sys.path,
            )
            _, stderr = process.communicate()
            result = process.returncode
            if result != 0:
                die(
                    "Failed to create pex via {}:\n{}".format(
                        " ".join(cmd), stderr.decode("utf-8")),
                    result,
                )
Esempio n. 19
0
 def _get_int(self, variable):
     # type: (str) -> int
     value = self._get_string(variable)
     try:
         return int(value)
     except ValueError:
         die("Invalid value for %s, must be an integer, got %r" %
             (variable, self._environ[variable]))
Esempio n. 20
0
 def to_python_interpreter(full_path_or_basename):
   if os.path.exists(full_path_or_basename):
     return PythonInterpreter.from_binary(full_path_or_basename)
   else:
     interpreter = PythonInterpreter.from_env(full_path_or_basename)
     if interpreter is None:
       die('Failed to find interpreter: %s' % full_path_or_basename)
     return interpreter
Esempio n. 21
0
File: pex.py Progetto: jneuff/pex
 def to_python_interpreter(full_path_or_basename):
     if os.path.isfile(full_path_or_basename):
         return PythonInterpreter.from_binary(full_path_or_basename)
     else:
         interp = PythonInterpreter.from_env(full_path_or_basename)
         if interp is None:
             die("Failed to find interpreter: %s" % full_path_or_basename)
         return interp
Esempio n. 22
0
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None):
    compatible_interpreters = _filter_compatible_interpreters(
        _find_pex_python(pex_python),
        compatibility_constraints=compatibility_constraints)
    if not compatible_interpreters:
        die('Failed to find a compatible PEX_PYTHON={} for constraints: {}'.
            format(pex_python, compatibility_constraints))
    return _select_interpreter(compatible_interpreters)
Esempio n. 23
0
File: pex.py Progetto: jsirois/pex
 def to_python_interpreter(full_path_or_basename):
   if os.path.exists(full_path_or_basename):
     return PythonInterpreter.from_binary(full_path_or_basename)
   else:
     interpreter = PythonInterpreter.from_env(full_path_or_basename)
     if interpreter is None:
       die('Failed to find interpreter: %s' % full_path_or_basename)
     return interpreter
Esempio n. 24
0
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None):
    compatible_interpreters_iter = _compatible_interpreters_iter(
        _iter_pex_python(pex_python),
        compatibility_constraints=compatibility_constraints)
    selected = _select_interpreter(compatible_interpreters_iter)
    if not selected:
        die('Failed to find a compatible PEX_PYTHON={} for constraints: {}'.
            format(pex_python, compatibility_constraints))
    return selected
Esempio n. 25
0
def validate_constraints(constraints):
  # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted:
  # https://github.com/pantsbuild/pex/issues/432
  for req in constraints:
    # Check that the compatibility requirements are well-formed.
    try:
      PythonIdentity.parse_requirement(req)
    except ValueError as e:
      die("Compatibility requirements are not formatted properly: %s" % str(e))
Esempio n. 26
0
 def _get_int(self, variable, default=None):
     # type: (str, Optional[int]) -> Optional[int]
     try:
         return int(self._environ[variable])
     except ValueError:
         die("Invalid value for %s, must be an integer, got %r" %
             (variable, self._environ[variable]))
     except KeyError:
         return self._defaulted(default)  # type: ignore[return-value]
Esempio n. 27
0
def validate_constraints(constraints):
  # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted:
  # https://github.com/pantsbuild/pex/issues/432
  for req in constraints:
    # Check that the compatibility requirements are well-formed.
    try:
      PythonIdentity.parse_requirement(req)
    except ValueError as e:
      die("Compatibility requirements are not formatted properly: %s" % str(e))
Esempio n. 28
0
def resolve_or_die(interpreter, requirement, options):
    resolve = functools.partial(resolve_interpreter,
                                options.interpreter_cache_dir, options.repos)

    interpreter = resolve(interpreter, requirement)
    if interpreter is None:
        die(
            'Could not find compatible interpreter that meets requirement %s' %
            requirement, CANNOT_SETUP_INTERPRETER)
    return interpreter
Esempio n. 29
0
 def _maybe_get_bool(self, variable):
     # type: (str) -> Optional[bool]
     value = self._maybe_get_string(variable)
     if value is None:
         return None
     if value.lower() in ("0", "false"):
         return False
     if value.lower() in ("1", "true"):
         return True
     die("Invalid value for %s, must be 0/1/false/true, got %r" % (variable, value))
Esempio n. 30
0
def _select_path_interpreter(path=None, compatibility_constraints=None):
    compatible_interpreters_iter = iter_compatible_interpreters(
        path=path, compatibility_constraints=compatibility_constraints)
    try:
        return _select_interpreter(compatible_interpreters_iter)
    except UnsatisfiableInterpreterConstraintsError as e:
        die(
            e.create_message(
                "Failed to find compatible interpreter on path {path}.".format(
                    path=path or os.getenv("PATH"))))
Esempio n. 31
0
 def _get_bool(self, variable, default=False):
   value = self._environ.get(variable)
   if value is not None:
     if value.lower() in ('0', 'false'):
       return False
     elif value.lower() in ('1', 'true'):
       return True
     else:
       die('Invalid value for %s, must be 0/1/false/true, got %r' % (variable, value))
   else:
     return self._defaulted(default)
Esempio n. 32
0
 def _get_bool(self, variable, default=False):
   value = self._environ.get(variable)
   if value is not None:
     if value.lower() in ('0', 'false'):
       return False
     elif value.lower() in ('1', 'true'):
       return True
     else:
       die('Invalid value for %s, must be 0/1/false/true, got %r' % (variable, value))
   else:
     return self._defaulted(default)
Esempio n. 33
0
 def _get_bool(self, variable, default=False):
     # type: (str, Optional[bool]) -> Optional[bool]
     value = self._environ.get(variable)
     if value is None:
         return self._defaulted(default)  # type: ignore[return-value]
     if value.lower() in ("0", "false"):
         return False
     if value.lower() in ("1", "true"):
         return True
     die("Invalid value for %s, must be 0/1/false/true, got %r" %
         (variable, value))
Esempio n. 34
0
 def _get_bool(self, variable, default=False):
     value = self._environ.get(variable)
     if value is not None:
         if value.lower() in ("0", "false"):
             return False
         elif value.lower() in ("1", "true"):
             return True
         else:
             die("Invalid value for %s, must be 0/1/false/true, got %r" % (variable, value))
     else:
         return self._defaulted(default)
Esempio n. 35
0
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None):
    compatible_interpreters_iter = _compatible_interpreters_iter(
        _iter_pex_python(pex_python),
        compatibility_constraints=compatibility_constraints)
    try:
        return _select_interpreter(compatible_interpreters_iter)
    except UnsatisfiableInterpreterConstraintsError as e:
        die(
            e.create_message(
                "Failed to find a compatible PEX_PYTHON={pex_python}.".format(
                    pex_python=pex_python)))
Esempio n. 36
0
def main(args=None):
    args = args[:] if args else sys.argv[1:]
    args = [transform_legacy_arg(arg) for arg in args]
    parser = configure_clp()

    try:
        separator = args.index('--')
        args, cmdline = args[:separator], args[separator + 1:]
    except ValueError:
        args, cmdline = args, []

    options, reqs = parser.parse_args(args=args)
    if options.python and options.interpreter_constraint:
        die('The "--python" and "--interpreter-constraint" options cannot be used together.'
            )

    with ENV.patch(PEX_VERBOSE=str(options.verbosity),
                   PEX_ROOT=options.pex_root) as patched_env:

        # Don't alter cache if it is disabled.
        if options.cache_dir:
            options.cache_dir = make_relative_to_root(options.cache_dir)

        with TRACER.timed('Building pex'):
            pex_builder = build_pex(reqs, options)

        pex_builder.freeze(bytecode_compile=options.compile)
        pex = PEX(pex_builder.path(),
                  interpreter=pex_builder.interpreter,
                  verify_entry_point=options.validate_ep)

        if options.pex_name is not None:
            log('Saving PEX file to %s' % options.pex_name,
                V=options.verbosity)
            tmp_name = options.pex_name + '~'
            safe_delete(tmp_name)
            pex_builder.build(
                tmp_name,
                bytecode_compile=options.compile,
                deterministic_timestamp=not options.use_system_time)
            os.rename(tmp_name, options.pex_name)
        else:
            if not _compatible_with_current_platform(options.platforms):
                log('WARNING: attempting to run PEX with incompatible platforms!',
                    V=1)
                log('Running on platform {} but built for {}'.format(
                    Platform.current(), ', '.join(map(str,
                                                      options.platforms))),
                    V=1)

            log('Running PEX file at %s with args %s' %
                (pex_builder.path(), cmdline),
                V=options.verbosity)
            sys.exit(pex.run(args=list(cmdline), env=patched_env))
Esempio n. 37
0
def _select_interpreter(pex_python_path=None, compatibility_constraints=None):
  compatible_interpreters = find_compatible_interpreters(
    pex_python_path=pex_python_path, compatibility_constraints=compatibility_constraints)

  if not compatible_interpreters:
    die('Failed to find compatible interpreter for constraints: %s'
        % str(compatibility_constraints))
  # TODO: https://github.com/pantsbuild/pex/issues/430
  target = min(compatible_interpreters).binary

  if os.path.exists(target):
    return target
Esempio n. 38
0
def main(args=None):
    args = args[:] if args else sys.argv[1:]
    args = [transform_legacy_arg(arg) for arg in args]
    parser, resolver_options_builder = configure_clp()

    try:
        separator = args.index('--')
        args, cmdline = args[:separator], args[separator + 1:]
    except ValueError:
        args, cmdline = args, []

    options, reqs = parser.parse_args(args=args)
    if options.python and options.interpreter_constraint:
        die('The "--python" and "--interpreter-constraint" options cannot be used together.'
            )

    if options.pex_root:
        ENV.set('PEX_ROOT', options.pex_root)
    else:
        options.pex_root = ENV.PEX_ROOT  # If option not specified fallback to env variable.

    # Don't alter cache if it is disabled.
    if options.cache_dir:
        options.cache_dir = make_relative_to_root(options.cache_dir)
    options.interpreter_cache_dir = make_relative_to_root(
        options.interpreter_cache_dir)

    with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
        with TRACER.timed('Building pex'):
            pex_builder = build_pex(reqs, options, resolver_options_builder)

        pex_builder.freeze()
        pex = PEX(pex_builder.path(),
                  interpreter=pex_builder.interpreter,
                  verify_entry_point=options.validate_ep)

        if options.pex_name is not None:
            log('Saving PEX file to %s' % options.pex_name,
                v=options.verbosity)
            tmp_name = options.pex_name + '~'
            safe_delete(tmp_name)
            pex_builder.build(tmp_name)
            os.rename(tmp_name, options.pex_name)
        else:
            if not _compatible_with_current_platform(options.platforms):
                log('WARNING: attempting to run PEX with incompatible platforms!'
                    )

            log('Running PEX file at %s with args %s' %
                (pex_builder.path(), cmdline),
                v=options.verbosity)
            sys.exit(pex.run(args=list(cmdline)))
Esempio n. 39
0
  def run(self):
    parser, options_builder = configure_clp()
    options, reqs = parser.parse_args(self.pex_args)

    if options.entry_point or options.script or options.pex_name:
      die('Must not specify entry point, script or output file to --pex-args, given: {}'
          .format(' '.join(self.pex_args)))

    name = self.distribution.get_name()
    version = self.distribution.get_version()

    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
      self.distribution.script_name)))
    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    console_scripts = self.parse_entry_points()

    pex_specs = []
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name))
                       for script_name in console_scripts)
    else:
      target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
      pex_specs.append((name if name in console_scripts else None, target))

    # In order for code to run to here, pex is on the sys.path - make sure to propagate the
    # sys.path so the subprocess can find us.
    env = os.environ.copy()
    env['PYTHONPATH'] = os.pathsep.join(sys.path)

    args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args
    if self.get_log_level() < log.INFO and options.verbosity == 0:
      args.append('-v')

    for script_name, target in pex_specs:
      cmd = args + ['--output-file', target]
      if script_name:
        log.info('Writing %s to %s' % (script_name, target))
        cmd += ['--script', script_name]
      else:
        # The package has no namesake entry point, so build an environment pex.
        log.info('Writing environment pex into %s' % target)

      log.debug('Building pex via: {}'.format(' '.join(cmd)))
      process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
      _, stderr = process.communicate()
      result = process.returncode
      if result != 0:
        die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')),
            result)
Esempio n. 40
0
File: pex.py Progetto: jsirois/pex
def main(args=None):
  args = args[:] if args else sys.argv[1:]
  args = [transform_legacy_arg(arg) for arg in args]
  parser, resolver_options_builder = configure_clp()

  try:
    separator = args.index('--')
    args, cmdline = args[:separator], args[separator + 1:]
  except ValueError:
    args, cmdline = args, []

  options, reqs = parser.parse_args(args=args)
  if options.python and options.interpreter_constraint:
    die('The "--python" and "--interpreter-constraint" options cannot be used together.')

  if options.pex_root:
    ENV.set('PEX_ROOT', options.pex_root)
  else:
    options.pex_root = ENV.PEX_ROOT  # If option not specified fallback to env variable.

  # Don't alter cache if it is disabled.
  if options.cache_dir:
    options.cache_dir = make_relative_to_root(options.cache_dir)

  with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
    with TRACER.timed('Building pex'):
      pex_builder = build_pex(reqs, options, resolver_options_builder)

    pex_builder.freeze(bytecode_compile=options.compile)
    pex = PEX(pex_builder.path(),
              interpreter=pex_builder.interpreter,
              verify_entry_point=options.validate_ep)

    if options.pex_name is not None:
      log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
      tmp_name = options.pex_name + '~'
      safe_delete(tmp_name)
      pex_builder.build(
        tmp_name,
        bytecode_compile=options.compile,
        deterministic_timestamp=not options.use_system_time
      )
      os.rename(tmp_name, options.pex_name)
    else:
      if not _compatible_with_current_platform(options.platforms):
        log('WARNING: attempting to run PEX with incompatible platforms!')

      log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
          V=options.verbosity)
      sys.exit(pex.run(args=list(cmdline)))
Esempio n. 41
0
File: pex.py Progetto: jsirois/pex
  def execute_content(cls, name, content, argv0=None):
    argv0 = argv0 or name
    try:
      ast = compile(content, name, 'exec', flags=0, dont_inherit=1)
    except SyntaxError:
      die('Unable to parse %s. PEX script support only supports Python scripts.' % name)

    cls.demote_bootstrap()

    from pex.compatibility import exec_function
    sys.argv[0] = argv0
    globals_map = globals().copy()
    globals_map['__name__'] = '__main__'
    globals_map['__file__'] = name
    exec_function(ast, globals_map)
Esempio n. 42
0
def build_pex(args):
    with TRACER.timed('Resolving interpreter', V=2):
        interpreter = _establish_interpreter(args)

    if interpreter is None:
        die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

    pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE)

    pex_info = pex_builder.info

    pex_info.zip_safe = False
    pex_info.always_write_cache = True
    pex_info.inherit_path = False

    resolver_option_builder = _establish_resolver_options(args)
    reqs = args.reqs
    resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs]

    for requirements_txt in args.requirement_files:
        resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

    resolver_kwargs = dict(interpreter=interpreter, platform=args.platform)
    _add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder)

    if not args.disable_cache:
        resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs)
    else:
        resolver = Resolver(**resolver_kwargs)

    resolveds = []
    with TRACER.timed('Resolving distributions'):
        try:
            resolveds = resolver.resolve(resolvables)
        except Unsatisfiable as exception:
            die(exception)

    for dist in resolveds:
        log('  %s' % dist, verbose=args.verbosity)
        pex_builder.add_distribution(dist)
        pex_builder.add_requirement(dist.as_requirement())

    pex_builder.set_entry_point('spex:spex')

    if args.python_shebang:
        pex_builder.set_shebang(args.python_shebang)

    return pex_builder
Esempio n. 43
0
File: pex.py Progetto: ewdurbin/pex
def build_pex(args, options, resolver_option_builder):
  with TRACER.timed('Resolving interpreter', V=2):
    interpreter = interpreter_from_options(options)

  if interpreter is None:
    die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

  pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)

  pex_info = pex_builder.info
  pex_info.zip_safe = options.zip_safe
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.inherit_path = options.inherit_path

  resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]

  for requirements_txt in options.requirement_files:
    resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

  resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)

  if options.cache_dir:
    resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
  else:
    resolver = Resolver(**resolver_kwargs)

  with TRACER.timed('Resolving distributions'):
    resolveds = resolver.resolve(resolvables)

  for dist in resolveds:
    log('  %s' % dist, v=options.verbosity)
    pex_builder.add_distribution(dist)
    pex_builder.add_requirement(dist.as_requirement())

  if options.entry_point and options.script:
    die('Must specify at most one entry point or script.', INVALID_OPTIONS)

  if options.entry_point:
    pex_builder.set_entry_point(options.entry_point)
  elif options.script:
    pex_builder.set_script(options.script)

  if options.python_shebang:
    pex_builder.set_shebang(options.python_shebang)

  return pex_builder
Esempio n. 44
0
def main(args=None):
    parser = create_parser()
    args = parser.parse_args(args)

    if args.build_distro:
        if not args.spark_home:
            die("No spark home given but building a distribution")

        spark_home = os.path.realpath(os.path.abspath(args.spark_home))
        if not os.path.exists(spark_home):
            die("No spark home given but building a distribution")
        spark_name = os.path.basename(spark_home)

        args.spark_home = spark_home
        args.spark_name = spark_name
    else:
        spark_home = None
        spark_name = None

    spex_name = args.spex_name
    spex_file = spex_name + '.spex'

    with ENV.patch(PEX_VERBOSE=str(args.verbosity)):
        with TRACER.timed('Building spex'):
            with TRACER.timed('Building pex'):
                pex_builder = build_pex(args)

                with dump_args_as_config(args) as cfg:
                    pex_builder.add_resource(cfg, 'SPEX-INFO')

                    log('Saving PEX file to %s' % spex_file, verbose=args.verbosity)
                    tmp_name = args.spex_name + '~'
                    safe_delete(tmp_name)
                    pex_builder.build(tmp_name)
                    os.rename(tmp_name, spex_file)

            if args.build_distro:
                with TRACER.timed('Building spark package'):
                    spark_distro = uber_distro_location(spark_name)
                    establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name)
                log('Spark package built')

                with TRACER.timed('Building full distribution'):
                    create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args)
                log('Saved full distribution to %s' % spark_distro)

    return 0
Esempio n. 45
0
  def run(self):
    name = self.distribution.get_name()
    version = self.distribution.get_version()
    parser, options_builder = configure_clp()
    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
        self.distribution.script_name)))

    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    options, reqs = parser.parse_args(self.pex_args)

    if options.entry_point or options.script:
      die('Must not specify entry_point or script to --pex-args')

    reqs = [package_dir] + reqs

    with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
      pex_builder = build_pex(reqs, options, options_builder)

    def split_and_strip(entry_point):
      console_script, entry_point = entry_point.split('=', 2)
      return console_script.strip(), entry_point.strip()

    try:
      console_scripts = dict(split_and_strip(script)
          for script in self.distribution.entry_points.get('console_scripts', []))
    except ValueError:
      console_scripts = {}

    target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      for script_name in console_scripts:
        target = os.path.join(self.bdist_dir, script_name)
        log.info('Writing %s to %s' % (script_name, target))
        self._write(pex_builder, target, script=script_name)
    elif name in console_scripts:
      # The package has a namesake entry point, so use it.
      log.info('Writing %s to %s' % (name, target))
      self._write(pex_builder, target, script=name)
    else:
      # The package has no namesake entry point, so build an environment pex.
      log.info('Writing environment pex into %s' % target)
      self._write(pex_builder, target, script=None)
Esempio n. 46
0
  def _resolve(self, working_set, reqs):
    reqs = reqs[:]
    unresolved_reqs = set()
    resolveds = set()

    environment = self._target_interpreter_env.copy()
    environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs))))

    # Resolve them one at a time so that we can figure out which ones we need to elide should
    # there be an interpreter incompatibility.
    for req in reqs:
      if req.marker and not req.marker.evaluate(environment=environment):
        TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
        continue
      with TRACER.timed('Resolving %s' % req, V=2):
        try:
          resolveds.update(working_set.resolve([req], env=self))
        except DistributionNotFound as e:
          TRACER.log('Failed to resolve a requirement: %s' % e)
          unresolved_reqs.add(e.req.project_name)
          if e.requirers:
            unresolved_reqs.update(e.requirers)

    unresolved_reqs = set([req.lower() for req in unresolved_reqs])

    if unresolved_reqs:
      TRACER.log('Unresolved requirements:')
      for req in unresolved_reqs:
        TRACER.log('  - %s' % req)
      TRACER.log('Distributions contained within this pex:')
      if not self._pex_info.distributions:
        TRACER.log('  None')
      else:
        for dist in self._pex_info.distributions:
          TRACER.log('  - %s' % dist)
      if not self._pex_info.ignore_errors:
        die(
          'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % (
            Platform.current(),
            '\n'.join(str(r) for r in unresolved_reqs)
          )
        )

    return resolveds
Esempio n. 47
0
  def run(self):
    name = self.distribution.get_name()
    version = self.distribution.get_version()
    parser, options_builder = configure_clp()
    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
        self.distribution.script_name)))

    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    options, reqs = parser.parse_args(self.pex_args)

    # Update cache_dir with pex_root in case this is being called directly.
    if options.cache_dir:
      options.cache_dir = make_relative_to_root(options.cache_dir)
    options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)

    if options.entry_point or options.script:
      die('Must not specify entry_point or script to --pex-args')

    reqs = [package_dir] + reqs

    with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root):
      pex_builder = build_pex(reqs, options, options_builder)

    console_scripts = self.parse_entry_points()

    target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      for script_name in console_scripts:
        target = os.path.join(self.bdist_dir, script_name)
        log.info('Writing %s to %s' % (script_name, target))
        self._write(pex_builder, target, script=script_name)
    elif name in console_scripts:
      # The package has a namesake entry point, so use it.
      log.info('Writing %s to %s' % (name, target))
      self._write(pex_builder, target, script=name)
    else:
      # The package has no namesake entry point, so build an environment pex.
      log.info('Writing environment pex into %s' % target)
      self._write(pex_builder, target, script=None)
Esempio n. 48
0
  def parse_entry_points(self):
    def parse_entry_point_name(entry_point):
      script_name = entry_point.split('=', 1)[0]
      return script_name.strip()

    raw_entry_points = self.distribution.entry_points

    if isinstance(raw_entry_points, string):
      parser = ConfigParser()
      parser.readfp(StringIO(to_unicode(raw_entry_points)))
      if parser.has_section('console_scripts'):
        return tuple(parser.options('console_scripts'))
    elif isinstance(raw_entry_points, dict):
      try:
        return tuple(parse_entry_point_name(script)
                     for script in raw_entry_points.get('console_scripts', []))
      except ValueError:
        pass
    elif raw_entry_points is not None:
      die('When entry_points is provided, it must be a string or dict.')

    return ()
Esempio n. 49
0
def _establish_interpreter(args):
    if args.python:
        if os.path.exists(args.python):
            interpreter = PythonInterpreter.from_binary(args.python)
        else:
            interpreter = PythonInterpreter.from_env(args.python)
        if interpreter is None:
            die('Failed to find interpreter: %s' % args.python)
    else:
        interpreter = PythonInterpreter.get()

    with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
        resolve = functools.partial(resolve_interpreter, args.interpreter_cache_dir, args.repos)

        # resolve setuptools
        interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)

        # possibly resolve wheel
        if interpreter and args.use_wheel:
            interpreter = resolve(interpreter, WHEEL_REQUIREMENT)

        return interpreter
Esempio n. 50
0
  def parse_entry_points(self):
    def split_and_strip(entry_point):
      console_script, entry_point = entry_point.split('=', 2)
      return console_script.strip(), entry_point.strip()

    raw_entry_points = self.distribution.entry_points

    if isinstance(raw_entry_points, string):
      parser = ConfigParser()
      parser.readfp(StringIO(raw_entry_points))
      if parser.has_section('console_scripts'):
        return dict(parser.items('console_scripts'))
    elif isinstance(raw_entry_points, dict):
      try:
        return dict(split_and_strip(script)
            for script in raw_entry_points.get('console_scripts', []))
      except ValueError:
        pass
    elif raw_entry_points is not None:
      die('When entry_points is provided, it must be a string or dict.')

    return {}
Esempio n. 51
0
def create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args):
    with tarfile.open(spex_name + '-distro.tar.bz2', 'w:bz2') as tarball:
        log('Including spex file [%s] in full distribution' % spex_file)
        tarball.add(spex_file, arcname=os.path.join(spex_name, spex_file))
        log('Including spark package [%s] in full distribution' % spark_distro)
        tarball.add(spark_distro, arcname=os.path.join(spex_name, spark_name + '.tar.bz2'))

        additional_artifacts = (
            ('files', args.files),
            ('py_files', args.py_files),
            ('jars', args.jars))

        log('Including additional artifacts in full distribution')
        for arcdir, artifacts in additional_artifacts:
            for artifact in (os.path.realpath(a) for a in artifacts):
                arcfile = os.path.basename(artifact)
                if os.path.exists(artifact):
                    arcname = os.path.join(os.path.join(spex_name, arcdir), arcfile)
                    log('Including ' + arcname)
                    tarball.add(artifact, arcname=arcname)
                else:
                    die('You asked me to include a %s that does not exist [%s]' % (arcdir, artifact))
Esempio n. 52
0
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
  interpreter = None

  if python_interpreter:
    if os.path.exists(python_interpreter):
      interpreter = PythonInterpreter.from_binary(python_interpreter)
    else:
      interpreter = PythonInterpreter.from_env(python_interpreter)
    if interpreter is None:
      die('Failed to find interpreter: %s' % python_interpreter)
  else:
    interpreter = PythonInterpreter.get()

  with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
    resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)

    # resolve setuptools
    interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)

    # possibly resolve wheel
    if interpreter and use_wheel:
      interpreter = resolve(interpreter, WHEEL_REQUIREMENT)

    return interpreter
Esempio n. 53
0
File: pex.py Progetto: twitter/heron
def interpreter_from_options(options):
    interpreter = None

    if options.python:
        if os.path.exists(options.python):
            interpreter = PythonInterpreter.from_binary(options.python)
        else:
            interpreter = PythonInterpreter.from_env(options.python)
        if interpreter is None:
            die("Failed to find interpreter: %s" % options.python)
    else:
        interpreter = PythonInterpreter.get()

    with TRACER.timed("Setting up interpreter %s" % interpreter.binary, V=2):
        resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)

        # resolve setuptools
        interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)

        # possibly resolve wheel
        if interpreter and options.use_wheel:
            interpreter = resolve(interpreter, WHEEL_REQUIREMENT)

        return interpreter
Esempio n. 54
0
def _select_pex_python_interpreter(target_python, compatibility_constraints=None):
  target = find_in_path(target_python)

  if not target:
    die('Failed to find interpreter specified by PEX_PYTHON: %s' % target)
  if compatibility_constraints:
    pi = PythonInterpreter.from_binary(target)
    if not list(matched_interpreters([pi], compatibility_constraints)):
      die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified '
          'interpreter constraints: %s' % (target, str(compatibility_constraints)))
  if not os.path.exists(target):
    die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target)
  return target
Esempio n. 55
0
def build_pex(args, options, resolver_option_builder):
  with TRACER.timed('Resolving interpreters', V=2):
    interpreters = [
      get_interpreter(interpreter,
                      options.interpreter_cache_dir,
                      options.repos,
                      options.use_wheel)
      for interpreter in options.python or [None]
    ]

  if not interpreters:
    die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

  try:
    with open(options.preamble_file) as preamble_fd:
      preamble = preamble_fd.read()
  except TypeError:
    # options.preamble_file is None
    preamble = None

  interpreter = _lowest_version_interpreter(interpreters)
  pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)

  pex_info = pex_builder.info
  pex_info.zip_safe = options.zip_safe
  pex_info.pex_path = options.pex_path
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.inherit_path = options.inherit_path

  resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]

  for requirements_txt in options.requirement_files:
    resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

  # pip states the constraints format is identical tor requirements
  # https://pip.pypa.io/en/stable/user_guide/#constraints-files
  for constraints_txt in options.constraint_files:
    constraints = []
    for r in requirements_from_file(constraints_txt, resolver_option_builder):
      r.is_constraint = True
      constraints.append(r)
    resolvables.extend(constraints)

  with TRACER.timed('Resolving distributions'):
    try:
      resolveds = resolve_multi(resolvables,
                                interpreters=interpreters,
                                platforms=options.platform,
                                cache=options.cache_dir,
                                cache_ttl=options.cache_ttl,
                                allow_prereleases=resolver_option_builder.prereleases_allowed)

      for dist in resolveds:
        log('  %s' % dist, v=options.verbosity)
        pex_builder.add_distribution(dist)
        pex_builder.add_requirement(dist.as_requirement())
    except Unsatisfiable as e:
      die(e)

  if options.entry_point and options.script:
    die('Must specify at most one entry point or script.', INVALID_OPTIONS)

  if options.entry_point:
    pex_builder.set_entry_point(options.entry_point)
  elif options.script:
    pex_builder.set_script(options.script)

  if options.python_shebang:
    pex_builder.set_shebang(options.python_shebang)

  return pex_builder
Esempio n. 56
0
File: pex.py Progetto: jsirois/pex
def build_pex(args, options, resolver_option_builder):
  with TRACER.timed('Resolving interpreters', V=2):
    def to_python_interpreter(full_path_or_basename):
      if os.path.exists(full_path_or_basename):
        return PythonInterpreter.from_binary(full_path_or_basename)
      else:
        interpreter = PythonInterpreter.from_env(full_path_or_basename)
        if interpreter is None:
          die('Failed to find interpreter: %s' % full_path_or_basename)
        return interpreter

    interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]

  if options.interpreter_constraint:
    # NB: options.python and interpreter constraints cannot be used together, so this will not
    # affect usages of the interpreter(s) specified by the "--python" command line flag.
    constraints = options.interpreter_constraint
    validate_constraints(constraints)
    if options.rc_file or not ENV.PEX_IGNORE_RCFILES:
      rc_variables = Variables.from_rc(rc=options.rc_file)
      pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
    else:
      pex_python_path = ""
    interpreters = find_compatible_interpreters(pex_python_path, constraints)

  if not interpreters:
    die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

  try:
    with open(options.preamble_file) as preamble_fd:
      preamble = preamble_fd.read()
  except TypeError:
    # options.preamble_file is None
    preamble = None

  interpreter = min(interpreters)

  pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)

  def walk_and_do(fn, src_dir):
    src_dir = os.path.normpath(src_dir)
    for root, dirs, files in os.walk(src_dir):
      for f in files:
        src_file_path = os.path.join(root, f)
        dst_path = os.path.relpath(src_file_path, src_dir)
        fn(src_file_path, dst_path)

  for directory in options.sources_directory:
    walk_and_do(pex_builder.add_source, directory)

  for directory in options.resources_directory:
    walk_and_do(pex_builder.add_resource, directory)

  pex_info = pex_builder.info
  pex_info.zip_safe = options.zip_safe
  pex_info.pex_path = options.pex_path
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.emit_warnings = options.emit_warnings
  pex_info.inherit_path = options.inherit_path
  if options.interpreter_constraint:
    for ic in options.interpreter_constraint:
      pex_builder.add_interpreter_constraint(ic)

  resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)

  for requirements_txt in options.requirement_files:
    resolvables.extend(requirements_from_file(requirements_txt,
                                              builder=resolver_option_builder,
                                              interpreter=interpreter))

  # pip states the constraints format is identical tor requirements
  # https://pip.pypa.io/en/stable/user_guide/#constraints-files
  for constraints_txt in options.constraint_files:
    constraints = []
    for r in requirements_from_file(constraints_txt,
                                    builder=resolver_option_builder,
                                    interpreter=interpreter):
      r.is_constraint = True
      constraints.append(r)
    resolvables.extend(constraints)

  with TRACER.timed('Resolving distributions'):
    try:
      resolveds = resolve_multi(resolvables,
                                interpreters=interpreters,
                                platforms=options.platforms,
                                cache=options.cache_dir,
                                cache_ttl=options.cache_ttl,
                                allow_prereleases=resolver_option_builder.prereleases_allowed,
                                use_manylinux=options.use_manylinux)

      for resolved_dist in resolveds:
        log('  %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
            V=options.verbosity)
        pex_builder.add_distribution(resolved_dist.distribution)
        pex_builder.add_requirement(resolved_dist.requirement)
    except Unsatisfiable as e:
      die(e)

  if options.entry_point and options.script:
    die('Must specify at most one entry point or script.', INVALID_OPTIONS)

  if options.entry_point:
    pex_builder.set_entry_point(options.entry_point)
  elif options.script:
    pex_builder.set_script(options.script)

  if options.python_shebang:
    pex_builder.set_shebang(options.python_shebang)

  return pex_builder