def spawn_install_wheel(self,
wheel,
install_dir,
compile=False,
overwrite=False,
cache=None,
target=None):
target = target or DistributionTarget.current()
install_cmd = [
'install', '--no-deps', '--no-index', '--only-binary', ':all:',
'--target', install_dir
]
interpreter = target.get_interpreter()
if target.is_foreign:
if compile:
raise ValueError(
'Cannot compile bytecode for {} using {} because the wheel has a foreign '
'platform.'.format(wheel, interpreter))
# We're installing a wheel for a foreign platform. This is just an unpacking operation though;
# so we don't actually need to perform it with a target platform compatible interpreter.
install_cmd.append('--ignore-requires-python')
install_cmd.append('--compile' if compile else '--no-compile')
if overwrite:
install_cmd.extend(['--upgrade', '--force-reinstall'])
install_cmd.append(wheel)
return self._spawn_pip_isolated(install_cmd,
cache=cache,
interpreter=interpreter)
def test_install():
# type: () -> None
project1_sdist = create_sdist(name="project1", version="1.0.0")
project2_wheel = build_wheel(name="project2", version="2.0.0")
installed_by_target = defaultdict(list)
for installed_distribution in install([
LocalDistribution.create(path=dist)
for dist in (project1_sdist, project2_wheel)
]):
installed_by_target[installed_distribution.target].append(
installed_distribution.distribution)
assert 1 == len(installed_by_target)
target, distributions = installed_by_target.popitem()
assert DistributionTarget.current() == target
distributions_by_name = {
distribution.key: distribution
for distribution in distributions
}
assert 2 == len(distributions_by_name)
assert "1.0.0" == distributions_by_name["project1"].version
assert "2.0.0" == distributions_by_name["project2"].version
assert 2 == len({
distribution.location
for distribution in distributions
}), "Expected installed distributions to have independent chroot paths."
def __init__(
self,
pex, # type: str
pex_info=None, # type: Optional[PexInfo]
target=None, # type: Optional[DistributionTarget]
):
# type: (...) -> None
self._pex = os.path.realpath(pex)
self._pex_info = pex_info or PexInfo.from_pex(pex)
self._available_ranked_dists_by_key = defaultdict(
list) # type: DefaultDict[str, List[_RankedDistribution]]
self._activated_dists = None # type: Optional[Iterable[Distribution]]
self._target = target or DistributionTarget.current()
self._interpreter_version = self._target.get_python_version_str()
# The supported_tags come ordered most specific (platform specific) to least specific
# (universal). We want to rank most specific highest; so we need to reverse iteration order
# here.
self._supported_tags_to_rank = {
tag: rank
for rank, tag in enumerate(
reversed(self._target.get_supported_tags()))
}
self._platform, _ = self._target.get_platform()
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._platform.impl == "pp" and zipfile.is_zipfile(self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
def test_install():
project1_sdist = create_sdist(name='project1', version='1.0.0')
project2_wheel = build_wheel(name='project2', version='2.0.0')
installed_by_target = defaultdict(list)
for installed_distribution in install([
LocalDistribution.create(path=dist)
for dist in (project1_sdist, project2_wheel)
]):
installed_by_target[installed_distribution.target].append(
installed_distribution.distribution)
assert 1 == len(installed_by_target)
target, distributions = installed_by_target.popitem()
assert DistributionTarget.current() == target
distributions_by_name = {
distribution.key: distribution
for distribution in distributions
}
assert 2 == len(distributions_by_name)
assert '1.0.0' == distributions_by_name['project1'].version
assert '2.0.0' == distributions_by_name['project2'].version
assert 2 == len({
distribution.location
for distribution in distributions
}), ('Expected installed distributions to have independent chroot paths.')
def spawn_install_wheel(self,
wheel,
install_dir,
compile=False,
cache=None,
target=None):
target = target or DistributionTarget.current()
install_cmd = [
"install",
"--no-deps",
"--no-index",
"--only-binary",
":all:",
"--target",
install_dir,
]
interpreter = target.get_interpreter()
if target.is_foreign:
if compile:
raise ValueError(
"Cannot compile bytecode for {} using {} because the wheel has a foreign "
"platform.".format(wheel, interpreter))
# We're installing a wheel for a foreign platform. This is just an unpacking operation though;
# so we don't actually need to perform it with a target platform compatible interpreter.
install_cmd.append("--ignore-requires-python")
install_cmd.append("--compile" if compile else "--no-compile")
install_cmd.append(wheel)
return self._spawn_pip_isolated(install_cmd,
cache=cache,
interpreter=interpreter)
def iter_targets():
if not interpreters and not parsed_platforms:
# No specified targets, so just build for the current interpreter (on the current platform).
yield DistributionTarget.current()
return
if interpreters:
for interpreter in interpreters:
# Build for the specified local interpreters (on the current platform).
yield DistributionTarget.for_interpreter(interpreter)
if parsed_platforms:
for platform in parsed_platforms:
if platform is not None or not interpreters:
# 1. Build for specific platforms.
# 2. Build for the current platform (None) only if not done already (ie: no intepreters
# were specified).
yield DistributionTarget.for_platform(platform)
def test_download():
# type: () -> None
project1_sdist = create_sdist(name="project1",
version="1.0.0",
extras_require={"foo": ["project2"]})
project2_wheel = build_wheel(
name="project2",
version="2.0.0",
# This is the last version of setuptools compatible with Python 2.7.
install_reqs=["setuptools==44.1.0"],
)
downloaded_by_target = defaultdict(list)
for local_distribution in download(
requirements=["{}[foo]".format(project1_sdist)],
find_links=[os.path.dirname(project2_wheel)],
):
distribution = pkginfo.get_metadata(local_distribution.path)
downloaded_by_target[local_distribution.target].append(distribution)
assert 1 == len(downloaded_by_target)
target, distributions = downloaded_by_target.popitem()
assert DistributionTarget.current() == target
distributions_by_name = {
distribution.name: distribution
for distribution in distributions
}
assert 3 == len(distributions_by_name)
def assert_dist(project_name, dist_type, version):
dist = distributions_by_name[project_name]
assert dist_type is type(dist)
assert version == dist.version
assert_dist("project1", pkginfo.SDist, "1.0.0")
assert_dist("project2", pkginfo.Wheel, "2.0.0")
assert_dist("setuptools", pkginfo.Wheel, "44.1.0")
def test_download():
project1_sdist = create_sdist(name='project1',
version='1.0.0',
extras_require={'foo': ['project2']})
project2_wheel = build_wheel(
name='project2',
version='2.0.0',
# This is the last version of setuptools compatible with Python 2.7.
install_reqs=['setuptools==44.1.0'])
downloaded_by_target = defaultdict(list)
for local_distribution in download(
requirements=['{}[foo]'.format(project1_sdist)],
find_links=[os.path.dirname(project2_wheel)]):
distribution = pkginfo.get_metadata(local_distribution.path)
downloaded_by_target[local_distribution.target].append(distribution)
assert 1 == len(downloaded_by_target)
target, distributions = downloaded_by_target.popitem()
assert DistributionTarget.current() == target
distributions_by_name = {
distribution.name: distribution
for distribution in distributions
}
assert 3 == len(distributions_by_name)
def assert_dist(project_name, dist_type, version):
dist = distributions_by_name[project_name]
assert dist_type is type(dist)
assert version == dist.version
assert_dist('project1', pkginfo.SDist, '1.0.0')
assert_dist('project2', pkginfo.Wheel, '2.0.0')
assert_dist('setuptools', pkginfo.Wheel, '44.1.0')
def create(cls, path, fingerprint=None, target=None):
fingerprint = fingerprint or fingerprint_path(path)
target = target or DistributionTarget.current()
return cls(target=target, path=path, fingerprint=fingerprint)
def spawn_download_distributions(self,
download_dir,
requirements=None,
requirement_files=None,
constraint_files=None,
allow_prereleases=False,
transitive=True,
target=None,
indexes=None,
find_links=None,
cache=None,
build=True,
manylinux=None,
use_wheel=True):
target = target or DistributionTarget.current()
platform = target.get_platform()
if not use_wheel:
if not build:
raise ValueError(
'Cannot both ignore wheels (use_wheel=False) and refrain from building '
'distributions (build=False).')
elif target.is_foreign:
raise ValueError(
'Cannot ignore wheels (use_wheel=False) when resolving for a foreign '
'platform: {}'.format(platform))
download_cmd = ['download', '--dest', download_dir]
package_index_options = self._calculate_package_index_options(
indexes=indexes, find_links=find_links)
download_cmd.extend(package_index_options)
if target.is_foreign:
# We're either resolving for a different host / platform or a different interpreter for the
# current platform that we have no access to; so we need to let pip know and not otherwise
# pickup platform info from the interpreter we execute pip with.
if manylinux and platform.platform.startswith('linux'):
download_cmd.extend([
'--platform',
platform.platform.replace('linux', manylinux, 1)
])
download_cmd.extend(['--platform', platform.platform])
download_cmd.extend(['--implementation', platform.impl])
download_cmd.extend(['--python-version', platform.version])
download_cmd.extend(['--abi', platform.abi])
if target.is_foreign or not build:
download_cmd.extend(['--only-binary', ':all:'])
if not use_wheel:
download_cmd.extend(['--no-binary', ':all:'])
if allow_prereleases:
download_cmd.append('--pre')
if not transitive:
download_cmd.append('--no-deps')
if requirement_files:
for requirement_file in requirement_files:
download_cmd.extend(['--requirement', requirement_file])
if constraint_files:
for constraint_file in constraint_files:
download_cmd.extend(['--constraint', constraint_file])
download_cmd.extend(requirements)
return self._spawn_pip_isolated(download_cmd,
cache=cache,
interpreter=target.get_interpreter())
def spawn_download_distributions(
self,
download_dir, # type: str
requirements=None, # type: Optional[Iterable[str]]
requirement_files=None, # type: Optional[Iterable[str]]
constraint_files=None, # type: Optional[Iterable[str]]
allow_prereleases=False, # type: bool
transitive=True, # type: bool
target=None, # type: Optional[DistributionTarget]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
build=True, # type: bool
manylinux=None, # type: Optional[str]
use_wheel=True, # type: bool
):
# type: (...) -> Job
target = target or DistributionTarget.current()
platform = target.get_platform()
if not use_wheel:
if not build:
raise ValueError(
"Cannot both ignore wheels (use_wheel=False) and refrain from building "
"distributions (build=False)."
)
elif target.is_foreign:
raise ValueError(
"Cannot ignore wheels (use_wheel=False) when resolving for a foreign "
"platform: {}".format(platform)
)
download_cmd = ["download", "--dest", download_dir]
if target.is_foreign:
# We're either resolving for a different host / platform or a different interpreter for
# the current platform that we have no access to; so we need to let pip know and not
# otherwise pickup platform info from the interpreter we execute pip with.
if manylinux and platform.platform.startswith("linux"):
download_cmd.extend(
["--platform", platform.platform.replace("linux", manylinux, 1)]
)
download_cmd.extend(["--platform", platform.platform])
download_cmd.extend(["--implementation", platform.impl])
download_cmd.extend(["--python-version", platform.version])
download_cmd.extend(["--abi", platform.abi])
if target.is_foreign or not build:
download_cmd.extend(["--only-binary", ":all:"])
if not use_wheel:
download_cmd.extend(["--no-binary", ":all:"])
if allow_prereleases:
download_cmd.append("--pre")
if not transitive:
download_cmd.append("--no-deps")
if requirement_files:
for requirement_file in requirement_files:
download_cmd.extend(["--requirement", requirement_file])
if constraint_files:
for constraint_file in constraint_files:
download_cmd.extend(["--constraint", constraint_file])
if requirements:
download_cmd.extend(requirements)
return self._spawn_pip_isolated(
download_cmd,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=target.get_interpreter(),
)
def spawn_install_wheel(
self,
wheel, # type: str
install_dir, # type: str
compile=False, # type: bool
cache=None, # type: Optional[str]
target=None, # type: Optional[DistributionTarget]
):
# type: (...) -> Job
target = target or DistributionTarget.current()
install_cmd = [
"install",
"--no-deps",
"--no-index",
"--only-binary",
":all:",
"--target",
install_dir,
]
interpreter = target.get_interpreter()
if target.is_foreign:
if compile:
raise ValueError(
"Cannot compile bytecode for {} using {} because the wheel has a foreign "
"platform.".format(wheel, interpreter))
# We're installing a wheel for a foreign platform. This is just an unpacking operation
# though; so we don't actually need to perform it with a target platform compatible
# interpreter (except for scripts - see below).
install_cmd.append("--ignore-requires-python")
# The new Pip 2020-resolver rightly refuses to install foreign wheels since they may
# contain python scripts that request a shebang re-write (see
# https://docs.python.org/3/distutils/setupscript.html#installing-scripts) in which case
# Pip would not be able to perform the re-write, leaving an un-runnable script. Since we
# only expose scripts via the Pex Venv tool and that tool re-writes shebangs anyhow, we
# trick Pip here by re-naming the wheel to look compatible with the current interpreter.
# Wheel filename format: https://www.python.org/dev/peps/pep-0427/#file-name-convention
# `{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl`
wheel_basename = os.path.basename(wheel)
wheel_name, extension = os.path.splitext(wheel_basename)
prefix, python_tag, abi_tag, platform_tag = wheel_name.rsplit(
"-", 3)
target_tags = PythonInterpreter.get().identity.supported_tags[0]
renamed_wheel = os.path.join(
os.path.dirname(wheel),
"{prefix}-{target_tags}{extension}".format(
prefix=prefix,
target_tags=target_tags,
extension=extension),
)
os.symlink(wheel_basename, renamed_wheel)
TRACER.log(
"Re-named {} to {} to perform foreign wheel install.".format(
wheel, renamed_wheel))
wheel = renamed_wheel
install_cmd.append("--compile" if compile else "--no-compile")
install_cmd.append(wheel)
return self._spawn_pip_isolated(install_cmd,
cache=cache,
interpreter=interpreter)
def spawn_download_distributions(
self,
download_dir, # type: str
requirements=None, # type: Optional[Iterable[str]]
requirement_files=None, # type: Optional[Iterable[str]]
constraint_files=None, # type: Optional[Iterable[str]]
allow_prereleases=False, # type: bool
transitive=True, # type: bool
target=None, # type: Optional[DistributionTarget]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
build=True, # type: bool
use_wheel=True, # type: bool
):
# type: (...) -> Job
target = target or DistributionTarget.current()
platform, manylinux = target.get_platform()
if not use_wheel:
if not build:
raise ValueError(
"Cannot both ignore wheels (use_wheel=False) and refrain from building "
"distributions (build=False).")
elif target.is_foreign:
raise ValueError(
"Cannot ignore wheels (use_wheel=False) when resolving for a foreign "
"platform: {}".format(platform))
download_cmd = ["download", "--dest", download_dir]
if target.is_foreign:
# We're either resolving for a different host / platform or a different interpreter for
# the current platform that we have no access to; so we need to let pip know and not
# otherwise pickup platform info from the interpreter we execute pip with.
download_cmd.extend(
self._iter_platform_args(
platform=platform.platform,
impl=platform.impl,
version=platform.version,
abi=platform.abi,
manylinux=manylinux,
))
if target.is_foreign or not build:
download_cmd.extend(["--only-binary", ":all:"])
if not use_wheel:
download_cmd.extend(["--no-binary", ":all:"])
if allow_prereleases:
download_cmd.append("--pre")
if not transitive:
download_cmd.append("--no-deps")
if requirement_files:
for requirement_file in requirement_files:
download_cmd.extend(["--requirement", requirement_file])
if constraint_files:
for constraint_file in constraint_files:
download_cmd.extend(["--constraint", constraint_file])
if requirements:
download_cmd.extend(requirements)
# The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed
# with other information we want to suppress. We jump though some hoops here to get at that
# information and surface it on stderr. See: https://github.com/pypa/pip/issues/9420.
log = None
if (self._calculate_resolver_version(
package_index_configuration=package_index_configuration) ==
ResolverVersion.PIP_2020):
log = os.path.join(safe_mkdtemp(), "pip.log")
download_cmd = ["--log", log] + download_cmd
command, process = self._spawn_pip_isolated(
download_cmd,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=target.get_interpreter(),
)
return self._Issue9420Job(command, process, log) if log else Job(
command, process)
