def test_install(): project1_sdist = create_sdist(name='project1', version='1.0.0') project2_wheel = build_wheel(name='project2', version='2.0.0') installed_by_target = defaultdict(list) for installed_distribution in install([ LocalDistribution.create(path=dist) for dist in (project1_sdist, project2_wheel) ]): installed_by_target[installed_distribution.target].append( installed_distribution.distribution) assert 1 == len(installed_by_target) target, distributions = installed_by_target.popitem() assert DistributionTarget.current() == target distributions_by_name = { distribution.key: distribution for distribution in distributions } assert 2 == len(distributions_by_name) assert '1.0.0' == distributions_by_name['project1'].version assert '2.0.0' == distributions_by_name['project2'].version assert 2 == len({ distribution.location for distribution in distributions }), ('Expected installed distributions to have independent chroot paths.')
def spawn_install_wheel(self, wheel, install_dir, compile=False, overwrite=False, cache=None, target=None): target = target or DistributionTarget.current() install_cmd = [ 'install', '--no-deps', '--no-index', '--only-binary', ':all:', '--target', install_dir ] interpreter = target.get_interpreter() if target.is_foreign: if compile: raise ValueError( 'Cannot compile bytecode for {} using {} because the wheel has a foreign ' 'platform.'.format(wheel, interpreter)) # We're installing a wheel for a foreign platform. This is just an unpacking operation though; # so we don't actually need to perform it with a target platform compatible interpreter. install_cmd.append('--ignore-requires-python') install_cmd.append('--compile' if compile else '--no-compile') if overwrite: install_cmd.extend(['--upgrade', '--force-reinstall']) install_cmd.append(wheel) return self._spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter)
def add_dist_location(self, dist, name=None): """Add a distribution by its location on disk. :param dist: The path to the distribution to add. :keyword name: (optional) The name of the distribution, should the dist directory alone be ambiguous. Packages contained within site-packages directories may require specifying ``name``. :raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution. PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution supported by setuptools/pkg_resources. """ self._ensure_unfrozen("Adding a distribution") dist_path = dist if os.path.isfile(dist_path) and dist_path.endswith(".whl"): dist_path = os.path.join(safe_mkdtemp(), os.path.basename(dist)) get_pip().spawn_install_wheel( wheel=dist, install_dir=dist_path, target=DistributionTarget.for_interpreter(self.interpreter), ).wait() dist = DistributionHelper.distribution_from_path(dist_path) self.add_distribution(dist, dist_name=name) self.add_requirement(dist.as_requirement())
def _activate(self): # type: () -> Iterable[Distribution] # set up the local .pex environment pex_info = self.pex_info() target = DistributionTarget.for_interpreter(self._interpreter) self._envs.append(PEXEnvironment(self._pex, pex_info, target=target)) # N.B. by this point, `pex_info.pex_path` will contain a single pex path # merged from pex_path in `PEX-INFO` and `PEX_PATH` set in the environment. # `PEX_PATH` entries written into `PEX-INFO` take precedence over those set # in the environment. if pex_info.pex_path: # set up other environments as specified in pex_path for pex_path in filter(None, pex_info.pex_path.split(os.pathsep)): pex_info = PexInfo.from_pex(pex_path) pex_info.update(self._pex_info_overrides) self._envs.append(PEXEnvironment(pex_path, pex_info, target=target)) # activate all of them activated_dists = [] # type: List[Distribution] for env in self._envs: activated_dists.extend(env.activate()) # Ensure that pkg_resources is not imported until at least every pex environment # (i.e. PEX_PATH) has been merged into the environment PEXEnvironment._declare_namespace_packages(activated_dists) return activated_dists
def test_install(): # type: () -> None project1_sdist = create_sdist(name="project1", version="1.0.0") project2_wheel = build_wheel(name="project2", version="2.0.0") installed_by_target = defaultdict(list) for installed_distribution in install([ LocalDistribution.create(path=dist) for dist in (project1_sdist, project2_wheel) ]): installed_by_target[installed_distribution.target].append( installed_distribution.distribution) assert 1 == len(installed_by_target) target, distributions = installed_by_target.popitem() assert DistributionTarget.current() == target distributions_by_name = { distribution.key: distribution for distribution in distributions } assert 2 == len(distributions_by_name) assert "1.0.0" == distributions_by_name["project1"].version assert "2.0.0" == distributions_by_name["project2"].version assert 2 == len({ distribution.location for distribution in distributions }), "Expected installed distributions to have independent chroot paths."
def __init__( self, pex, # type: str pex_info=None, # type: Optional[PexInfo] target=None, # type: Optional[DistributionTarget] ): # type: (...) -> None self._pex = os.path.realpath(pex) self._pex_info = pex_info or PexInfo.from_pex(pex) self._available_ranked_dists_by_key = defaultdict( list) # type: DefaultDict[str, List[_RankedDistribution]] self._activated_dists = None # type: Optional[Iterable[Distribution]] self._target = target or DistributionTarget.current() self._interpreter_version = self._target.get_python_version_str() # The supported_tags come ordered most specific (platform specific) to least specific # (universal). We want to rank most specific highest; so we need to reverse iteration order # here. self._supported_tags_to_rank = { tag: rank for rank, tag in enumerate( reversed(self._target.get_supported_tags())) } self._platform, _ = self._target.get_platform() # For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686 # NB: This must be installed early before the underlying pex is loaded in any way. if self._platform.impl == "pp" and zipfile.is_zipfile(self._pex): self._install_pypy_zipimporter_workaround(self._pex)
def spawn_install_wheel(self, wheel, install_dir, compile=False, cache=None, target=None): target = target or DistributionTarget.current() install_cmd = [ "install", "--no-deps", "--no-index", "--only-binary", ":all:", "--target", install_dir, ] interpreter = target.get_interpreter() if target.is_foreign: if compile: raise ValueError( "Cannot compile bytecode for {} using {} because the wheel has a foreign " "platform.".format(wheel, interpreter)) # We're installing a wheel for a foreign platform. This is just an unpacking operation though; # so we don't actually need to perform it with a target platform compatible interpreter. install_cmd.append("--ignore-requires-python") install_cmd.append("--compile" if compile else "--no-compile") install_cmd.append(wheel) return self._spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter)
def _add_dist_wheel_file(self, path, dist_name): with temporary_dir() as install_dir: get_pip().spawn_install_wheel( wheel=path, install_dir=install_dir, target=DistributionTarget.for_interpreter(self.interpreter), ).wait() return self._add_dist_dir(install_dir, dist_name)
def iter_targets(): if not interpreters and not parsed_platforms: # No specified targets, so just build for the current interpreter (on the current platform). yield DistributionTarget.current() return if interpreters: for interpreter in interpreters: # Build for the specified local interpreters (on the current platform). yield DistributionTarget.for_interpreter(interpreter) if parsed_platforms: for platform in parsed_platforms: if platform is not None or not interpreters: # 1. Build for specific platforms. # 2. Build for the current platform (None) only if not done already (ie: no intepreters # were specified). yield DistributionTarget.for_platform(platform)
def make_bdist(name='my_project', version='0.0.0', zip_safe=True, interpreter=None, **kwargs): with built_wheel(name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget.for_interpreter(interpreter) ).wait() yield DistributionHelper.distribution_from_path(install_dir)
def _loaded_envs(self): # type: () -> Iterable[PEXEnvironment] if self._envs is None: # set up the local .pex environment pex_info = self.pex_info() target = DistributionTarget.for_interpreter(self._interpreter) envs = [PEXEnvironment(self._pex, pex_info, target=target)] # N.B. by this point, `pex_info.pex_path` will contain a single pex path # merged from pex_path in `PEX-INFO` and `PEX_PATH` set in the environment. # `PEX_PATH` entries written into `PEX-INFO` take precedence over those set # in the environment. if pex_info.pex_path: # set up other environments as specified in pex_path for pex_path in filter(None, pex_info.pex_path.split(os.pathsep)): pex_info = PexInfo.from_pex(pex_path) pex_info.update(self._pex_info_overrides) envs.append( PEXEnvironment(pex_path, pex_info, target=target)) self._envs = tuple(envs) return self._envs
def make_bdist( name="my_project", # type: str version="0.0.0", # type: str zip_safe=True, # type: bool interpreter=None, # type: Optional[PythonInterpreter] **kwargs # type: Any ): # type: (...) -> Iterator[Distribution] with built_wheel( name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs ) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget.for_interpreter(interpreter), ).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert dist is not None yield dist
def test_download(): # type: () -> None project1_sdist = create_sdist(name="project1", version="1.0.0", extras_require={"foo": ["project2"]}) project2_wheel = build_wheel( name="project2", version="2.0.0", # This is the last version of setuptools compatible with Python 2.7. install_reqs=["setuptools==44.1.0"], ) downloaded_by_target = defaultdict(list) for local_distribution in download( requirements=["{}[foo]".format(project1_sdist)], find_links=[os.path.dirname(project2_wheel)], ): distribution = pkginfo.get_metadata(local_distribution.path) downloaded_by_target[local_distribution.target].append(distribution) assert 1 == len(downloaded_by_target) target, distributions = downloaded_by_target.popitem() assert DistributionTarget.current() == target distributions_by_name = { distribution.name: distribution for distribution in distributions } assert 3 == len(distributions_by_name) def assert_dist(project_name, dist_type, version): dist = distributions_by_name[project_name] assert dist_type is type(dist) assert version == dist.version assert_dist("project1", pkginfo.SDist, "1.0.0") assert_dist("project2", pkginfo.Wheel, "2.0.0") assert_dist("setuptools", pkginfo.Wheel, "44.1.0")
def test_download(): project1_sdist = create_sdist(name='project1', version='1.0.0', extras_require={'foo': ['project2']}) project2_wheel = build_wheel( name='project2', version='2.0.0', # This is the last version of setuptools compatible with Python 2.7. install_reqs=['setuptools==44.1.0']) downloaded_by_target = defaultdict(list) for local_distribution in download( requirements=['{}[foo]'.format(project1_sdist)], find_links=[os.path.dirname(project2_wheel)]): distribution = pkginfo.get_metadata(local_distribution.path) downloaded_by_target[local_distribution.target].append(distribution) assert 1 == len(downloaded_by_target) target, distributions = downloaded_by_target.popitem() assert DistributionTarget.current() == target distributions_by_name = { distribution.name: distribution for distribution in distributions } assert 3 == len(distributions_by_name) def assert_dist(project_name, dist_type, version): dist = distributions_by_name[project_name] assert dist_type is type(dist) assert version == dist.version assert_dist('project1', pkginfo.SDist, '1.0.0') assert_dist('project2', pkginfo.Wheel, '2.0.0') assert_dist('setuptools', pkginfo.Wheel, '44.1.0')
def spawn_download_distributions( self, download_dir, # type: str requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] constraint_files=None, # type: Optional[Iterable[str]] allow_prereleases=False, # type: bool transitive=True, # type: bool target=None, # type: Optional[DistributionTarget] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] cache=None, # type: Optional[str] build=True, # type: bool use_wheel=True, # type: bool ): # type: (...) -> Job target = target or DistributionTarget.current() platform, manylinux = target.get_platform() if not use_wheel: if not build: raise ValueError( "Cannot both ignore wheels (use_wheel=False) and refrain from building " "distributions (build=False).") elif target.is_foreign: raise ValueError( "Cannot ignore wheels (use_wheel=False) when resolving for a foreign " "platform: {}".format(platform)) download_cmd = ["download", "--dest", download_dir] if target.is_foreign: # We're either resolving for a different host / platform or a different interpreter for # the current platform that we have no access to; so we need to let pip know and not # otherwise pickup platform info from the interpreter we execute pip with. download_cmd.extend( self._iter_platform_args( platform=platform.platform, impl=platform.impl, version=platform.version, abi=platform.abi, manylinux=manylinux, )) if target.is_foreign or not build: download_cmd.extend(["--only-binary", ":all:"]) if not use_wheel: download_cmd.extend(["--no-binary", ":all:"]) if allow_prereleases: download_cmd.append("--pre") if not transitive: download_cmd.append("--no-deps") if requirement_files: for requirement_file in requirement_files: download_cmd.extend(["--requirement", requirement_file]) if constraint_files: for constraint_file in constraint_files: download_cmd.extend(["--constraint", constraint_file]) if requirements: download_cmd.extend(requirements) # The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed # with other information we want to suppress. We jump though some hoops here to get at that # information and surface it on stderr. See: https://github.com/pypa/pip/issues/9420. log = None if (self._calculate_resolver_version( package_index_configuration=package_index_configuration) == ResolverVersion.PIP_2020): log = os.path.join(safe_mkdtemp(), "pip.log") download_cmd = ["--log", log] + download_cmd command, process = self._spawn_pip_isolated( download_cmd, package_index_configuration=package_index_configuration, cache=cache, interpreter=target.get_interpreter(), ) return self._Issue9420Job(command, process, log) if log else Job( command, process)
def spawn_download_distributions( self, download_dir, # type: str requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] constraint_files=None, # type: Optional[Iterable[str]] allow_prereleases=False, # type: bool transitive=True, # type: bool target=None, # type: Optional[DistributionTarget] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] cache=None, # type: Optional[str] build=True, # type: bool manylinux=None, # type: Optional[str] use_wheel=True, # type: bool ): # type: (...) -> Job target = target or DistributionTarget.current() platform = target.get_platform() if not use_wheel: if not build: raise ValueError( "Cannot both ignore wheels (use_wheel=False) and refrain from building " "distributions (build=False)." ) elif target.is_foreign: raise ValueError( "Cannot ignore wheels (use_wheel=False) when resolving for a foreign " "platform: {}".format(platform) ) download_cmd = ["download", "--dest", download_dir] if target.is_foreign: # We're either resolving for a different host / platform or a different interpreter for # the current platform that we have no access to; so we need to let pip know and not # otherwise pickup platform info from the interpreter we execute pip with. if manylinux and platform.platform.startswith("linux"): download_cmd.extend( ["--platform", platform.platform.replace("linux", manylinux, 1)] ) download_cmd.extend(["--platform", platform.platform]) download_cmd.extend(["--implementation", platform.impl]) download_cmd.extend(["--python-version", platform.version]) download_cmd.extend(["--abi", platform.abi]) if target.is_foreign or not build: download_cmd.extend(["--only-binary", ":all:"]) if not use_wheel: download_cmd.extend(["--no-binary", ":all:"]) if allow_prereleases: download_cmd.append("--pre") if not transitive: download_cmd.append("--no-deps") if requirement_files: for requirement_file in requirement_files: download_cmd.extend(["--requirement", requirement_file]) if constraint_files: for constraint_file in constraint_files: download_cmd.extend(["--constraint", constraint_file]) if requirements: download_cmd.extend(requirements) return self._spawn_pip_isolated( download_cmd, package_index_configuration=package_index_configuration, cache=cache, interpreter=target.get_interpreter(), )
def resolve(requirements=None, requirement_files=None, constraint_files=None, allow_prereleases=False, transitive=True, interpreter=None, platform=None, indexes=None, find_links=None, cache=None, build=True, use_wheel=True, compile=False, manylinux=None, max_parallel_jobs=None, ignore_errors=False): """Produce all distributions needed to meet all specified requirements. :keyword requirements: A sequence of requirement strings. :type requirements: list of str :keyword requirement_files: A sequence of requirement file paths. :type requirement_files: list of str :keyword constraint_files: A sequence of constraint file paths. :type constraint_files: list of str :keyword bool allow_prereleases: Whether to include pre-release and development versions when resolving requirements. Defaults to ``False``, but any requirements that explicitly request prerelease or development versions will override this setting. :keyword bool transitive: Whether to resolve transitive dependencies of requirements. Defaults to ``True``. :keyword interpreter: The interpreter to use for building distributions and for testing distribution compatibility. Defaults to the current interpreter. :type interpreter: :class:`pex.interpreter.PythonInterpreter` :keyword str platform: The exact target platform to resolve distributions for. If ``None`` or ``'current'``, resolve for distributions appropriate for `interpreter`. :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search for distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn off use of all indexes, pass an empty list. :type indexes: list of str :keyword find_links: A list or URLs, paths to local html files or directory paths. If URLs or local html file paths, these are parsed for links to distributions. If a local directory path, its listing is used to discover distributons. :type find_links: list of str :keyword str cache: A directory path to use to cache distributions locally. :keyword bool build: Whether to allow building source distributions when no wheel is found. Defaults to ``True``. :keyword bool use_wheel: Whether to allow resolution of pre-built wheel distributions. Defaults to ``True``. :keyword bool compile: Whether to pre-compile resolved distribution python sources. Defaults to ``False``. :keyword str manylinux: The upper bound manylinux standard to support when targeting foreign linux platforms. Defaults to ``None``. :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. """ target = DistributionTarget(interpreter=interpreter, platform=parsed_platform(platform)) resolve_request = ResolveRequest(targets=[target], requirements=requirements, requirement_files=requirement_files, constraint_files=constraint_files, allow_prereleases=allow_prereleases, transitive=transitive, indexes=indexes, find_links=find_links, cache=cache, build=build, use_wheel=use_wheel, compile=compile, manylinux=manylinux, max_parallel_jobs=max_parallel_jobs) return list( resolve_request.resolve_distributions(ignore_errors=ignore_errors))
def cpython_35_environment(python_35_interpreter): return PEXEnvironment( pex="", pex_info=PexInfo.default(python_35_interpreter), target=DistributionTarget.for_interpreter(python_35_interpreter), )
def create(cls, path, fingerprint=None, target=None): fingerprint = fingerprint or fingerprint_path(path) target = target or DistributionTarget.current() return cls(target=target, path=path, fingerprint=fingerprint)
def spawn_install_wheel( self, wheel, # type: str install_dir, # type: str compile=False, # type: bool cache=None, # type: Optional[str] target=None, # type: Optional[DistributionTarget] ): # type: (...) -> Job target = target or DistributionTarget.current() install_cmd = [ "install", "--no-deps", "--no-index", "--only-binary", ":all:", "--target", install_dir, ] interpreter = target.get_interpreter() if target.is_foreign: if compile: raise ValueError( "Cannot compile bytecode for {} using {} because the wheel has a foreign " "platform.".format(wheel, interpreter)) # We're installing a wheel for a foreign platform. This is just an unpacking operation # though; so we don't actually need to perform it with a target platform compatible # interpreter (except for scripts - see below). install_cmd.append("--ignore-requires-python") # The new Pip 2020-resolver rightly refuses to install foreign wheels since they may # contain python scripts that request a shebang re-write (see # https://docs.python.org/3/distutils/setupscript.html#installing-scripts) in which case # Pip would not be able to perform the re-write, leaving an un-runnable script. Since we # only expose scripts via the Pex Venv tool and that tool re-writes shebangs anyhow, we # trick Pip here by re-naming the wheel to look compatible with the current interpreter. # Wheel filename format: https://www.python.org/dev/peps/pep-0427/#file-name-convention # `{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl` wheel_basename = os.path.basename(wheel) wheel_name, extension = os.path.splitext(wheel_basename) prefix, python_tag, abi_tag, platform_tag = wheel_name.rsplit( "-", 3) target_tags = PythonInterpreter.get().identity.supported_tags[0] renamed_wheel = os.path.join( os.path.dirname(wheel), "{prefix}-{target_tags}{extension}".format( prefix=prefix, target_tags=target_tags, extension=extension), ) os.symlink(wheel_basename, renamed_wheel) TRACER.log( "Re-named {} to {} to perform foreign wheel install.".format( wheel, renamed_wheel)) wheel = renamed_wheel install_cmd.append("--compile" if compile else "--no-compile") install_cmd.append(wheel) return self._spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter)
def spawn_download_distributions(self, download_dir, requirements=None, requirement_files=None, constraint_files=None, allow_prereleases=False, transitive=True, target=None, indexes=None, find_links=None, cache=None, build=True, manylinux=None, use_wheel=True): target = target or DistributionTarget.current() platform = target.get_platform() if not use_wheel: if not build: raise ValueError( 'Cannot both ignore wheels (use_wheel=False) and refrain from building ' 'distributions (build=False).') elif target.is_foreign: raise ValueError( 'Cannot ignore wheels (use_wheel=False) when resolving for a foreign ' 'platform: {}'.format(platform)) download_cmd = ['download', '--dest', download_dir] package_index_options = self._calculate_package_index_options( indexes=indexes, find_links=find_links) download_cmd.extend(package_index_options) if target.is_foreign: # We're either resolving for a different host / platform or a different interpreter for the # current platform that we have no access to; so we need to let pip know and not otherwise # pickup platform info from the interpreter we execute pip with. if manylinux and platform.platform.startswith('linux'): download_cmd.extend([ '--platform', platform.platform.replace('linux', manylinux, 1) ]) download_cmd.extend(['--platform', platform.platform]) download_cmd.extend(['--implementation', platform.impl]) download_cmd.extend(['--python-version', platform.version]) download_cmd.extend(['--abi', platform.abi]) if target.is_foreign or not build: download_cmd.extend(['--only-binary', ':all:']) if not use_wheel: download_cmd.extend(['--no-binary', ':all:']) if allow_prereleases: download_cmd.append('--pre') if not transitive: download_cmd.append('--no-deps') if requirement_files: for requirement_file in requirement_files: download_cmd.extend(['--requirement', requirement_file]) if constraint_files: for constraint_file in constraint_files: download_cmd.extend(['--constraint', constraint_file]) download_cmd.extend(requirements) return self._spawn_pip_isolated(download_cmd, cache=cache, interpreter=target.get_interpreter())