Esempio n. 1
0
File: pex.py Progetto: kamilchm/pex
def build_pex(args, options):
  interpreter = interpreter_from_options(options)

  pex_builder = PEXBuilder(
      path=safe_mkdtemp(),
      interpreter=interpreter,
  )

  pex_info = pex_builder.info

  pex_info.zip_safe = options.zip_safe
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.inherit_path = options.inherit_path

  installer = WheelInstaller if options.use_wheel else EggInstaller

  resolveds = requirement_resolver(
      options.requirements,
      obtainer=build_obtainer(options),
      interpreter=interpreter,
      platform=options.platform)

  if resolveds:
    log('Resolved distributions:', v=options.verbosity)

  for pkg in resolveds:
    log('  %s' % pkg, v=options.verbosity)
    pex_builder.add_distribution(pkg)
    pex_builder.add_requirement(pkg.as_requirement())

  for source_dir in options.source_dirs:
    try:
      bdist = installer(source_dir).bdist()
    except installer.Error:
      die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
    pex_builder.add_dist_location(bdist)

  if options.entry_point is not None:
    log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
    pex_builder.info.entry_point = options.entry_point
  else:
    log('Creating environment PEX.', v=options.verbosity)

  return pex_builder
Esempio n. 2
0
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None):
  """Write a pex file that contains an executable entry point

  :param td: temporary directory path
  :param exe_contents: entry point python file
  :type exe_contents: string
  :param dists: distributions to include, typically sdists or bdists
  :param sources: sources to include, as a list of pairs (env_filename, contents)
  :param coverage: include coverage header
  :param interpreter: a custom interpreter to use to build the pex
  """
  dists = dists or []
  sources = sources or []

  safe_mkdir(td)

  with open(os.path.join(td, 'exe.py'), 'w') as fp:
    fp.write(exe_contents)

  pb = PEXBuilder(path=td,
                  preamble=COVERAGE_PREAMBLE if coverage else None,
                  interpreter=interpreter)

  for dist in dists:
    pb.add_dist_location(dist.location)

  for env_filename, contents in sources:
    src_path = os.path.join(td, env_filename)
    safe_mkdir(os.path.dirname(src_path))
    with open(src_path, 'w') as fp:
      fp.write(contents)
    pb.add_source(src_path, env_filename)

  pb.set_executable(os.path.join(td, 'exe.py'))
  pb.freeze()

  return pb
Esempio n. 3
0
    def create(cls, path):
        # type: (str) -> Pip
        """Creates a pip tool with PEX isolation at path.

        :param path: The path to build the pip tool pex at.
        """
        pip_pex_path = os.path.join(path, isolated().pex_hash)
        with atomic_directory(pip_pex_path, exclusive=True) as chroot:
            if chroot is not None:
                from pex.pex_builder import PEXBuilder

                isolated_pip_builder = PEXBuilder(path=chroot)
                for dist_location in third_party.expose(["pip", "setuptools", "wheel"]):
                    isolated_pip_builder.add_dist_location(dist=dist_location)
                with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp:
                    fp.write(
                        dedent(
                            """\
                            import os
                            import runpy
                            import sys
                            
                            
                            # Propagate un-vendored setuptools to pip for any legacy setup.py builds
                            # it needs to perform.
                            os.environ['__PEX_UNVENDORED__'] = '1'
                            os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
                            
                            runpy.run_module('pip', run_name='__main__')
                            """
                        )
                    )
                isolated_pip_builder.set_executable(fp.name)
                isolated_pip_builder.freeze()

        return cls(pip_pex_path)
Esempio n. 4
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
    parser.add_option('--python', default=sys.executable)
    parser.add_option('--preload', action='append', default=[])
    options, args = parser.parse_args()
    if len(args) == 1:
        output = args[0]
    else:
        parser.error("'output' positional argument is required")
        return 1

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    manifest = json.load(sys.stdin)

    # Setup a temp dir that the PEX builder will use as its scratch dir.
    tmp_dir = tempfile.mkdtemp()
    try:

        # The version of pkg_resources.py (from setuptools) on some distros is
        # too old for PEX.  So we keep a recent version in the buck repo and
        # force it into the process by constructing a custom PythonInterpreter
        # instance using it.
        interpreter = PythonInterpreter(
            options.python,
            PythonInterpreter.from_binary(options.python).identity,
            extras={})

        pex_builder = PEXBuilder(
            path=tmp_dir,
            interpreter=interpreter,
        )

        # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
        # and we'll rely on python's zip-import mechanism to load modules from
        # the PEX.  This may not work in some situations (e.g. native
        # libraries, libraries that want to find resources via the FS).
        pex_builder.info.zip_safe = options.zip_safe

        # Set the starting point for this PEX.
        pex_builder.info.entry_point = options.entry_point

        # Copy in our version of `pkg_resources` & `_markerlib`.
        copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
        copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)

        # Add the sources listed in the manifest.
        for dst, src in manifest['modules'].iteritems():
            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as e:
                raise Exception("Failed to add {}: {}".format(src, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['resources'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add prebuilt libraries listed in the manifest.
        for req in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(req)
            except Exception as e:
                raise Exception("Failed to add {}: {}".format(req, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['nativeLibraries'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Generate the PEX file.
        pex_builder.build(output)

    # Always try cleaning up the scratch dir, ignoring failures.
    finally:
        shutil.rmtree(tmp_dir, True)
Esempio n. 5
0
def yield_pex_builder(zip_safe=True, interpreter=None):
    # type: (bool, Optional[PythonInterpreter]) -> Iterator[PEXBuilder]
    with temporary_dir() as td, make_bdist("p1", zip_safe=zip_safe, interpreter=interpreter) as p1:
        pb = PEXBuilder(path=td, interpreter=interpreter)
        pb.add_dist_location(p1.location)
        yield pb
Esempio n. 6
0
class PexBuilderWrapper:
    """Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities."""
    class Factory(Subsystem):
        options_scope = "pex-builder-wrapper"

        @classmethod
        def register_options(cls, register):
            super(PexBuilderWrapper.Factory, cls).register_options(register)
            # TODO: make an analogy to cls.register_jvm_tool that can be overridden for python subsystems
            # by a python_requirement_library() target, not just via pants.ini!
            register(
                "--setuptools-version",
                advanced=True,
                default="40.6.3",
                fingerprint=True,
                help=
                "The setuptools version to include in the pex if namespace packages need "
                "to be injected.",
            )
            register(
                "--pex-version",
                advanced=True,
                default=pex_version,
                fingerprint=True,
                help="The pex version to include in any generated ipex files. "
                "NOTE: This should ideally be the same as the pex version which pants "
                f"itself depends on, which right now is {pex_version}.",
            )

        @classmethod
        def subsystem_dependencies(cls):
            return super(PexBuilderWrapper.Factory,
                         cls).subsystem_dependencies() + (
                             PythonRepos,
                             PythonSetup,
                         )

        @classmethod
        def create(cls, builder, log=None, generate_ipex=False):
            options = cls.global_instance().get_options()
            setuptools_requirement = f"setuptools=={options.setuptools_version}"
            pex_requirement = f"pex=={options.pex_version}"

            log = log or logging.getLogger(__name__)

            return PexBuilderWrapper(
                builder=builder,
                python_repos_subsystem=PythonRepos.global_instance(),
                python_setup_subsystem=PythonSetup.global_instance(),
                setuptools_requirement=PythonRequirement(
                    setuptools_requirement),
                pex_requirement=PythonRequirement(pex_requirement),
                log=log,
                generate_ipex=generate_ipex,
            )

    def __init__(
        self,
        builder: PEXBuilder,
        python_repos_subsystem: PythonRepos,
        python_setup_subsystem: PythonSetup,
        setuptools_requirement: PythonRequirement,
        pex_requirement: PythonRequirement,
        log,
        generate_ipex: bool = False,
    ):
        assert log is not None

        self._builder = builder
        self._python_repos_subsystem = python_repos_subsystem
        self._python_setup_subsystem = python_setup_subsystem
        self._setuptools_requirement = setuptools_requirement
        self._pex_requirement = pex_requirement
        self._log = log

        self._distributions: Dict[str, Distribution] = {}
        self._frozen = False

        self._generate_ipex = generate_ipex
        # If we generate a .ipex, we need to ensure all the code we copy into the underlying PEXBuilder
        # is also added to the new PEXBuilder created in `._shuffle_original_build_info_into_ipex()`.
        self._all_added_sources_resources: List[Path] = []
        # If we generate a dehydrated "ipex" file, we need to make sure that it is aware of any special
        # find_links repos attached to any single requirement, so it can later resolve those
        # requirements when it is first bootstrapped, using the same resolve options.
        self._all_find_links: OrderedSet[str] = OrderedSet()

    def add_requirement_libs_from(self, req_libs, platforms=None):
        """Multi-platform dependency resolution for PEX files.

        :param builder: Dump the requirements into this builder.
        :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
        :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
        :param log: Use this logger.
        :param platforms: A list of :class:`Platform`s to resolve requirements for.
                                            Defaults to the platforms specified by PythonSetup.
        """
        reqs = [req for req_lib in req_libs for req in req_lib.requirements]
        self.add_resolved_requirements(reqs, platforms=platforms)

    class SingleDistExtractionError(Exception):
        pass

    def extract_single_dist_for_current_platform(self, reqs,
                                                 dist_key) -> Distribution:
        """Resolve a specific distribution from a set of requirements matching the current platform.

        :param list reqs: A list of :class:`PythonRequirement` to resolve.
        :param str dist_key: The value of `distribution.key` to match for a `distribution` from the
                                                 resolved requirements.
        :return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
        :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the
                 given `dist_key`.
        """
        distributions = self.resolve_distributions(reqs, platforms=["current"])
        try:
            matched_dist = assert_single_element(
                dist for dists in distributions.values() for dist in dists
                if dist.key == dist_key)
        except (StopIteration, ValueError) as e:
            raise self.SingleDistExtractionError(
                f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
            )
        return matched_dist

    def resolve_distributions(
        self,
        reqs: List[PythonRequirement],
        platforms: Optional[List[Platform]] = None,
    ) -> Dict[str, List[Distribution]]:
        """Multi-platform dependency resolution.

        :param reqs: A list of :class:`PythonRequirement` to resolve.
        :param platforms: A list of platform strings to resolve requirements for.
                          Defaults to the platforms specified by PythonSetup.
        :returns: A tuple `(map, transitive_reqs)`, where `map` is a dict mapping distribution name
                  to a list of resolved distributions, and `reqs` contains all transitive ==
                  requirements
                  needed to resolve the initial given requirements `reqs` for the given platforms.
        """
        deduped_reqs = OrderedSet(reqs)
        find_links: OrderedSet[str] = OrderedSet()
        for req in deduped_reqs:
            self._log.debug(f"  Dumping requirement: {req}")
            self._builder.add_requirement(str(req.requirement))
            if req.repository:
                find_links.add(req.repository)

        # Resolve the requirements into distributions.
        distributions = self._resolve_multi(
            self._builder.interpreter,
            list(deduped_reqs),
            platforms,
            list(find_links),
        )
        return distributions

    def add_resolved_requirements(
        self,
        reqs: List[PythonRequirement],
        platforms: Optional[List[Platform]] = None,
        override_ipex_build_do_actually_add_distribution: bool = False,
    ) -> None:
        """Multi-platform dependency resolution for PEX files.

        :param builder: Dump the requirements into this builder.
        :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
        :param reqs: A list of :class:`PythonRequirement` to resolve.
        :param log: Use this logger.
        :param platforms: A list of :class:`Platform`s to resolve requirements for.
                                            Defaults to the platforms specified by PythonSetup.
        :param bool override_ipex_build_do_actually_add_distribution: When this PexBuilderWrapper is configured with
                                                                        generate_ipex=True, this method won't add any distributions to
                                                                        the output pex. The internal implementation of this class adds a
                                                                        pex dependency to the output ipex file, and therefore needs to
                                                                        override the default behavior of this method.
        """
        distributions = self.resolve_distributions(reqs, platforms=platforms)
        locations: Set[str] = set()
        for platform, dists in distributions.items():
            for dist in dists:
                if dist.location not in locations:
                    if self._generate_ipex and not override_ipex_build_do_actually_add_distribution:
                        self._log.debug(
                            f"  *AVOIDING* dumping distribution into ipex: .../{os.path.basename(dist.location)}"
                        )
                        self._register_distribution(dist)
                    else:
                        self._log.debug(
                            f"  Dumping distribution: .../{os.path.basename(dist.location)}"
                        )
                        self.add_distribution(dist)
                locations.add(dist.location)

    def _resolve_multi(
        self,
        interpreter: PythonInterpreter,
        requirements: List[PythonRequirement],
        platforms: Optional[List[Platform]],
        find_links: Optional[List[str]],
    ) -> Dict[str, List[Distribution]]:
        """Multi-platform dependency resolution for PEX files.

        Returns a tuple containing a list of distributions that must be included in order to satisfy a
        set of requirements, and the transitive == requirements for those distributions. This may
        involve distributions for multiple platforms.

        :param interpreter: The :class:`PythonInterpreter` to resolve for.
        :param requirements: A list of :class:`PythonRequirement` objects to resolve.
        :param platforms: A list of :class:`Platform`s to resolve for.
        :param find_links: Additional paths to search for source packages during resolution.
        :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
                         to satisfy the requirements on that platform.
        """
        python_setup = self._python_setup_subsystem
        python_repos = self._python_repos_subsystem
        platforms = platforms or python_setup.platforms

        find_links = list(find_links) if find_links else []
        find_links.extend(python_repos.repos)

        # Individual requirements from pants may have a `repository` link attached to them, which is
        # extracted in `self.resolve_distributions()`. When generating a .ipex file with
        # `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it
        # tries to resolve all the requirements from BOOTSTRAP-PEX-INFO.
        self._all_find_links.update(OrderedSet(find_links))

        distributions: Dict[str, List[Distribution]] = defaultdict(list)

        for platform in platforms:
            requirements_cache_dir = os.path.join(
                python_setup.resolver_cache_dir, str(interpreter.identity))
            resolved_dists = resolve(
                requirements=[str(req.requirement) for req in requirements],
                interpreter=interpreter,
                platform=platform,
                indexes=python_repos.indexes,
                find_links=find_links,
                cache=requirements_cache_dir,
                allow_prereleases=python_setup.resolver_allow_prereleases,
                manylinux=python_setup.manylinux,
            )
            for resolved_dist in resolved_dists:
                distributions[platform].append(resolved_dist.distribution)

        return distributions

    def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]:
        buildroot = get_buildroot()

        def get_chroot_path(relpath: str) -> str:
            if type(tgt) == Files:
                # Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package
                # structure and so we chroot them relative to the build root so that they can be accessed
                # via the normal Python filesystem APIs just as they would be accessed outside the
                # chrooted environment. NB: This requires we mark the pex as not zip safe so
                # these `Files` can still be accessed in the context of a built pex distribution.
                self._builder.info.zip_safe = False
                return relpath
            return str(Path(relpath).relative_to(tgt.target_base))

        def dump_source(relpath: str) -> None:
            source_path = str(Path(buildroot, relpath))
            dest_path = get_chroot_path(relpath)

            self._all_added_sources_resources.append(Path(dest_path))
            if has_resources(tgt):
                self._builder.add_resource(filename=source_path,
                                           env_filename=dest_path)
            else:
                self._builder.add_source(filename=source_path,
                                         env_filename=dest_path)

        return dump_source

    def add_sources_from(self, tgt: Target) -> None:
        dump_source = self._create_source_dumper(tgt)
        self._log.debug(f"  Dumping sources: {tgt}")
        for relpath in tgt.sources_relative_to_buildroot():
            try:
                dump_source(relpath)
            except OSError:
                self._log.error(
                    f"Failed to copy {relpath} for target {tgt.address.spec}")
                raise

        if getattr(tgt, "_resource_target_specs", None) or getattr(
                tgt, "_synthetic_resources_target", None):
            # No one should be on old-style resources any more.  And if they are,
            # switching to the new python pipeline will be a great opportunity to fix that.
            raise TaskError(
                f"Old-style resources not supported for target {tgt.address.spec}. Depend on resources() "
                "targets instead.")

    def _prepare_inits(self) -> Set[str]:
        chroot = self._builder.chroot()
        sources = chroot.get("source") | chroot.get("resource")
        missing_init_files = identify_missing_init_files(sources)
        if missing_init_files:
            with temporary_file(permissions=0o644) as ns_package:
                ns_package.write(
                    b'__import__("pkg_resources").declare_namespace(__name__)  # type: ignore[attr-defined]'
                )
                ns_package.flush()
                for missing_init_file in missing_init_files:
                    self._all_added_sources_resources.append(
                        Path(missing_init_file))
                    self._builder.add_source(filename=ns_package.name,
                                             env_filename=missing_init_file)
        return missing_init_files

    def set_emit_warnings(self, emit_warnings):
        self._builder.info.emit_warnings = emit_warnings

    def _set_major_minor_interpreter_constraint_for_ipex(
        self,
        info: PexInfo,
        identity: PythonIdentity,
    ) -> PexInfo:
        interpreter_name = identity.requirement.name
        major, minor, _patch = identity.version
        major_minor_only_constraint = f"{interpreter_name}=={major}.{minor}.*"
        return ipex_launcher.modify_pex_info(
            info, interpreter_constraints=[str(major_minor_only_constraint)])

    def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
        """Replace the original builder with a new one, and just pull files from the old chroot."""
        # Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is
        # (the exact same interpreter we used to resolve those requirements here). This is the only (?)
        # way to ensure that the ipex bootstrap uses the *exact* same interpreter version.
        self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
            self._builder.info, self._builder.interpreter.identity)

        # Remove all the original top-level requirements in favor of the transitive == requirements.
        self._builder.info = ipex_launcher.modify_pex_info(self._builder.info,
                                                           requirements=[])
        transitive_reqs = [
            dist.as_requirement() for dist in self._distributions.values()
        ]
        self.add_direct_requirements(transitive_reqs)

        orig_info = self._builder.info.copy()

        orig_chroot = self._builder.chroot()

        # Mutate the PexBuilder object which is manipulated by this subsystem.
        self._builder = PEXBuilder(interpreter=self._builder.interpreter)
        self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
            self._builder.info, self._builder.interpreter.identity)

        self._distributions = {}

        return (orig_info, Path(orig_chroot.path()))

    def _shuffle_original_build_info_into_ipex(self):
        """Create a "dehydrated" ipex file without any of its requirements, and specify that in two.

        *-INFO files.

        See ipex_launcher.py for details of how these files are used.
        """
        orig_pex_info, orig_chroot = self._shuffle_underlying_pex_builder()

        # Gather information needed to create IPEX-INFO.
        all_code = [str(src) for src in self._all_added_sources_resources]
        prefixed_code_paths = [
            os.path.join(ipex_launcher.APP_CODE_PREFIX, src)
            for src in all_code
        ]
        for src, prefixed in zip(all_code, prefixed_code_paths):
            # NB: Need to add under 'source' label for `self._prepare_inits()` to pick it up!
            self._builder.chroot().copy(os.path.join(str(orig_chroot), src),
                                        prefixed,
                                        label="source")

        python_repos = self._python_repos_subsystem
        python_setup = self._python_setup_subsystem

        # NB: self._all_find_links is updated on every call to self._resolve_multi(), and therefore
        # includes all of the links from python_repos.repos, as well as any links added within any
        # individual requirements from that resolve.

        resolver_settings = dict(
            indexes=list(python_repos.indexes),
            find_links=list(self._all_find_links),
            allow_prereleases=UnsetBool.coerce_bool(
                python_setup.resolver_allow_prereleases, default=True),
            manylinux=python_setup.manylinux,
        )

        # IPEX-INFO: A json mapping interpreted in ipex_launcher.py:
        # {
        #   "code": [<which source files to add to the "hydrated" pex when bootstrapped>],
        #   "resolver_settings": {<which indices to search for requirements from when bootstrapping>},
        # }
        ipex_info = dict(
            code=prefixed_code_paths,
            resolver_settings=resolver_settings,
        )
        with temporary_file(permissions=0o644) as ipex_info_file:
            ipex_info_file.write(json.dumps(ipex_info).encode())
            ipex_info_file.flush()
            self._builder.add_resource(filename=ipex_info_file.name,
                                       env_filename="IPEX-INFO")

        # BOOTSTRAP-PEX-INFO: The original PEX-INFO, which should be the PEX-INFO in the hydrated .pex
        #                     file that is generated when the .ipex is first executed.
        with temporary_file(permissions=0o644) as bootstrap_pex_info_file:
            bootstrap_pex_info_file.write(orig_pex_info.dump().encode())
            bootstrap_pex_info_file.flush()
            self._builder.add_resource(filename=bootstrap_pex_info_file.name,
                                       env_filename="BOOTSTRAP-PEX-INFO")

        # ipex.py: The special bootstrap script to hydrate the .ipex with the fully resolved
        #          requirements when it is first executed.
        # Extract the file contents of our custom app launcher script from the pants package.
        parent_module = module_dirname(module_dirname(ipex_launcher.__name__))
        ipex_launcher_provider = get_provider(parent_module)
        ipex_launcher_script = ipex_launcher_provider.get_resource_string(
            parent_module, "ipex/ipex_launcher.py")
        with temporary_file(permissions=0o644) as ipex_launcher_file:
            ipex_launcher_file.write(ipex_launcher_script)
            ipex_launcher_file.flush()
            # Our .ipex file will use our custom app launcher!
            self._builder.set_executable(ipex_launcher_file.name,
                                         env_filename="ipex.py")

        # The PEX-INFO we generate shouldn't have any requirements (except pex itself), or they will
        # fail to bootstrap because they were unable to find those distributions. Instead, the .pex file
        # produced when the .ipex is first executed will read and resolve all those requirements from
        # the BOOTSTRAP-PEX-INFO.
        self.add_resolved_requirements(
            [self._pex_requirement, self._setuptools_requirement],
            override_ipex_build_do_actually_add_distribution=True,
        )

    def freeze(self) -> None:
        if self._frozen:
            return

        if self._prepare_inits():
            dist = self._distributions.get("setuptools")
            if not dist:
                self.add_resolved_requirements([self._setuptools_requirement])

        if self._generate_ipex:
            self._shuffle_original_build_info_into_ipex()

        self._builder.freeze(bytecode_compile=False)
        self._frozen = True

    def set_entry_point(self, entry_point):
        self._builder.set_entry_point(entry_point)

    def build(self, safe_path):
        self.freeze()
        self._builder.build(safe_path,
                            bytecode_compile=False,
                            deterministic_timestamp=True)

    def set_shebang(self, shebang):
        self._builder.set_shebang(shebang)

    def add_interpreter_constraint(self, constraint):
        self._builder.add_interpreter_constraint(constraint)

    def add_interpreter_constraints_from(self, constraint_tgts):
        # TODO this would be a great place to validate the constraints and present a good error message
        # if they are incompatible because all the sources of the constraints are available.
        # See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py
        constraint_tuples = {
            self._python_setup_subsystem.compatibility_or_constraints(
                tgt.compatibility)
            for tgt in constraint_tgts
        }
        for constraint_tuple in constraint_tuples:
            for constraint in constraint_tuple:
                self.add_interpreter_constraint(constraint)

    def add_direct_requirements(self, reqs):
        for req in reqs:
            self._builder.add_requirement(str(req))

    def add_distribution(self, dist):
        self._builder.add_distribution(dist)
        self._register_distribution(dist)

    def add_dist_location(self, location):
        self._builder.add_dist_location(location)
        dist = DistributionHelper.distribution_from_path(location)
        self._register_distribution(dist)

    def _register_distribution(self, dist):
        self._distributions[dist.key] = dist

    def set_script(self, script):
        self._builder.set_script(script)
Esempio n. 7
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--directory', action='store_true', default=False)
    parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
    parser.add_option('--python', default='')
    parser.add_option('--python-version', default='')
    parser.add_option('--preload', action='append', default=[])
    options, args = parser.parse_args()
    if len(args) == 1:
        output = args[0]
    else:
        parser.error("'output' positional argument is required")
        return 1

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    manifest = json.load(sys.stdin)

    # The version of pkg_resources.py (from setuptools) on some distros is
    # too old for PEX.  So we keep a recent version in the buck repo and
    # force it into the process by constructing a custom PythonInterpreter
    # instance using it.
    if not options.python:
        options.python = sys.executable
        identity = PythonIdentity.get()
    elif not options.python_version:
        # Note: this is expensive (~500ms). prefer passing --python-version when possible.
        identity = PythonInterpreter.from_binary(options.python).identity
    else:
        # Convert "CPython 2.7" to "CPython 2 7 0"
        python_version = options.python_version.replace('.', ' ').split()
        if len(python_version) == 3:
            python_version.append('0')
        identity = PythonIdentity.from_id_string(' '.join(python_version))

    interpreter = PythonInterpreter(
        options.python,
        identity,
        extras={})

    pex_builder = PEXBuilder(
        path=output if options.directory else None,
        interpreter=interpreter,
    )

    # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
    # and we'll rely on python's zip-import mechanism to load modules from
    # the PEX.  This may not work in some situations (e.g. native
    # libraries, libraries that want to find resources via the FS).
    pex_builder.info.zip_safe = options.zip_safe

    # Set the starting point for this PEX.
    pex_builder.info.entry_point = options.entry_point

    # Copy in our version of `pkg_resources` & `_markerlib`.
    copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
    copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)

    # Add the sources listed in the manifest.
    for dst, src in manifest['modules'].iteritems():
        # NOTE(agallagher): calls the `add_source` and `add_resource` below
        # hard-link the given source into the PEX temp dir.  Since OS X and
        # Linux behave different when hard-linking a source that is a
        # symbolic link (Linux does *not* follow symlinks), resolve any
        # layers of symlinks here to get consistent behavior.
        try:
            pex_builder.add_source(dereference_symlinks(src), dst)
        except OSError as e:
            raise Exception("Failed to add {}: {}".format(src, e))

    # Add resources listed in the manifest.
    for dst, src in manifest['resources'].iteritems():
        # NOTE(agallagher): see rationale above.
        pex_builder.add_resource(dereference_symlinks(src), dst)

    # Add prebuilt libraries listed in the manifest.
    for req in manifest.get('prebuiltLibraries', []):
        try:
            pex_builder.add_dist_location(req)
        except Exception as e:
            raise Exception("Failed to add {}: {}".format(req, e))

    # Add resources listed in the manifest.
    for dst, src in manifest['nativeLibraries'].iteritems():
        # NOTE(agallagher): see rationale above.
        pex_builder.add_resource(dereference_symlinks(src), dst)

    if options.directory:
        pex_builder.freeze(code_hash=False, bytecode_compile=False)
    else:
        pex_builder.build(output)
Esempio n. 8
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--directory', action='store_true', default=False)
    parser.add_option('--no-zip-safe',
                      action='store_false',
                      dest='zip_safe',
                      default=True)
    parser.add_option('--python', default='')
    parser.add_option('--python-version', default='')
    parser.add_option('--preload', action='append', default=[])
    options, args = parser.parse_args()
    if len(args) == 1:
        output = args[0]
    else:
        parser.error("'output' positional argument is required")
        return 1

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    manifest = json.load(sys.stdin)

    # The version of pkg_resources.py (from setuptools) on some distros is
    # too old for PEX.  So we keep a recent version in the buck repo and
    # force it into the process by constructing a custom PythonInterpreter
    # instance using it.
    if not options.python:
        options.python = sys.executable
        identity = PythonIdentity.get()
    elif not options.python_version:
        # Note: this is expensive (~500ms). prefer passing --python-version when possible.
        identity = PythonInterpreter.from_binary(options.python).identity
    else:
        # Convert "CPython 2.7" to "CPython 2 7 0"
        python_version = options.python_version.replace('.', ' ').split()
        if len(python_version) == 3:
            python_version.append('0')
        identity = PythonIdentity.from_id_string(' '.join(python_version))

    interpreter = PythonInterpreter(options.python, identity, extras={})

    pex_builder = PEXBuilder(
        path=output if options.directory else None,
        interpreter=interpreter,
    )

    # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
    # and we'll rely on python's zip-import mechanism to load modules from
    # the PEX.  This may not work in some situations (e.g. native
    # libraries, libraries that want to find resources via the FS).
    pex_builder.info.zip_safe = options.zip_safe

    # Set the starting point for this PEX.
    pex_builder.info.entry_point = options.entry_point

    # Copy in our version of `pkg_resources` & `_markerlib`.
    copy_package(pex_builder,
                 'pkg_resources',
                 prefix=pex_builder.BOOTSTRAP_DIR)
    copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)

    # Add the sources listed in the manifest.
    for dst, src in manifest['modules'].iteritems():
        # NOTE(agallagher): calls the `add_source` and `add_resource` below
        # hard-link the given source into the PEX temp dir.  Since OS X and
        # Linux behave different when hard-linking a source that is a
        # symbolic link (Linux does *not* follow symlinks), resolve any
        # layers of symlinks here to get consistent behavior.
        try:
            pex_builder.add_source(dereference_symlinks(src), dst)
        except OSError as e:
            raise Exception("Failed to add {}: {}".format(src, e))

    # Add resources listed in the manifest.
    for dst, src in manifest['resources'].iteritems():
        # NOTE(agallagher): see rationale above.
        pex_builder.add_resource(dereference_symlinks(src), dst)

    # Add prebuilt libraries listed in the manifest.
    for req in manifest.get('prebuiltLibraries', []):
        try:
            pex_builder.add_dist_location(req)
        except Exception as e:
            raise Exception("Failed to add {}: {}".format(req, e))

    # Add resources listed in the manifest.
    for dst, src in manifest['nativeLibraries'].iteritems():
        # NOTE(agallagher): see rationale above.
        pex_builder.add_resource(dereference_symlinks(src), dst)

    if options.directory:
        pex_builder.freeze(code_hash=False, bytecode_compile=False)
    else:
        pex_builder.build(output)
Esempio n. 9
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--no-zip-safe',
                      action='store_false',
                      dest='zip_safe',
                      default=True)
    parser.add_option('--python', default=sys.executable)
    parser.add_option('--preload', action='append', default=[])
    options, args = parser.parse_args()
    if len(args) == 1:
        output = args[0]
    else:
        parser.error("'output' positional argument is required")
        return 1

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    manifest = json.load(sys.stdin)

    # Setup a temp dir that the PEX builder will use as its scratch dir.
    tmp_dir = tempfile.mkdtemp()
    try:

        # The version of pkg_resources.py (from setuptools) on some distros is
        # too old for PEX.  So we keep a recent version in the buck repo and
        # force it into the process by constructing a custom PythonInterpreter
        # instance using it.
        interpreter = PythonInterpreter(options.python,
                                        PythonInterpreter.from_binary(
                                            options.python).identity,
                                        extras={})

        pex_builder = PEXBuilder(
            path=tmp_dir,
            interpreter=interpreter,
        )

        # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
        # and we'll rely on python's zip-import mechanism to load modules from
        # the PEX.  This may not work in some situations (e.g. native
        # libraries, libraries that want to find resources via the FS).
        pex_builder.info.zip_safe = options.zip_safe

        # Set the starting point for this PEX.
        pex_builder.info.entry_point = options.entry_point

        # Copy in our version of `pkg_resources`.
        copy_package(pex_builder,
                     'pkg_resources',
                     prefix=pex_builder.BOOTSTRAP_DIR)

        # Add the sources listed in the manifest.
        for dst, src in manifest['modules'].iteritems():
            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as e:
                raise Exception("Failed to add {}: {}".format(src, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['resources'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add prebuilt libraries listed in the manifest.
        for req in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(req)
            except Exception as e:
                raise Exception("Failed to add {}: {}".format(req, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['nativeLibraries'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Generate the PEX file.
        pex_builder.build(output)

    # Always try cleaning up the scratch dir, ignoring failures.
    finally:
        shutil.rmtree(tmp_dir, True)
Esempio n. 10
0
def build_pex(args, options):
  interpreter = interpreter_from_options(options)

  pex_builder = PEXBuilder(
      path=safe_mkdtemp(),
      interpreter=interpreter,
  )

  pex_info = pex_builder.info

  pex_info.zip_safe = options.zip_safe
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.inherit_path = options.inherit_path

  installer = WheelInstaller if options.use_wheel else EggInstaller

  interpreter = interpreter_from_options(options)

  fetchers = [Fetcher(options.repos)]

  if options.pypi:
    fetchers.append(PyPIFetcher())

  if options.indices:
    fetchers.extend(PyPIFetcher(index) for index in options.indices)

  translator = translator_from_options(options)

  if options.use_wheel:
    precedence = (WheelPackage, EggPackage, SourcePackage)
  else:
    precedence = (EggPackage, SourcePackage)

  with TRACER.timed('Resolving distributions'):
    resolveds = requirement_resolver(
        options.requirements,
        fetchers=fetchers,
        translator=translator,
        interpreter=interpreter,
        platform=options.platform,
        precedence=precedence,
        cache=options.cache_dir,
        cache_ttl=options.cache_ttl)

  for pkg in resolveds:
    log('  %s' % pkg, v=options.verbosity)
    pex_builder.add_distribution(pkg)
    pex_builder.add_requirement(pkg.as_requirement())

  for source_dir in options.source_dirs:
    try:
      bdist = installer(source_dir).bdist()
    except installer.Error:
      die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
    pex_builder.add_dist_location(bdist)

  if options.entry_point is not None:
    log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
    pex_builder.info.entry_point = options.entry_point
  else:
    log('Creating environment PEX.', v=options.verbosity)

  return pex_builder