def installed_wheel(wheel_path):
# type: (str) -> Iterator[Distribution]
with temporary_dir() as install_dir:
get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None, "Could not load a distribution from {}.".format(install_dir)
yield dist
def add_dist_location(self, dist, name=None):
"""Add a distribution by its location on disk.
:param dist: The path to the distribution to add.
:keyword name: (optional) The name of the distribution, should the dist directory alone be
ambiguous. Packages contained within site-packages directories may require specifying
``name``.
:raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution.
PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution
supported by setuptools/pkg_resources.
"""
self._ensure_unfrozen("Adding a distribution")
dist_path = dist
if os.path.isfile(dist_path) and dist_path.endswith(".whl"):
dist_path = os.path.join(safe_mkdtemp(), os.path.basename(dist))
get_pip().spawn_install_wheel(
wheel=dist,
install_dir=dist_path,
target=DistributionTarget.for_interpreter(self.interpreter),
).wait()
dist = DistributionHelper.distribution_from_path(dist_path)
self.add_distribution(dist, dist_name=name)
self.add_requirement(dist.as_requirement())
def _add_dist_wheel_file(self, path, dist_name):
with temporary_dir() as install_dir:
get_pip().spawn_install_wheel(
wheel=path,
install_dir=install_dir,
target=DistributionTarget.for_interpreter(self.interpreter),
).wait()
return self._add_dist_dir(install_dir, dist_name)
def pip_wheel(pip_tgz_sdist):
# type: (str) -> Iterator[str]
with temporary_dir() as wheel_dir:
get_pip().spawn_build_wheels([pip_tgz_sdist], wheel_dir=wheel_dir).wait()
wheels = os.listdir(wheel_dir)
assert len(wheels) == 1, "Expected 1 wheel to be built for {}.".format(pip_tgz_sdist)
wheel = os.path.join(wheel_dir, wheels[0])
assert wheel.endswith(".whl")
yield wheel
def install_wheel(
wheel_path, # type: str
install_dir, # type: str
):
# type: (...) -> Distribution
get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None, "Could not load a distribution from {}".format(install_dir)
return dist
def resolved_distribution(requirement):
# type: (str) -> Distribution
with temporary_dir() as td:
download_dir = os.path.join(td, "download")
get_pip().spawn_download_distributions(
download_dir=download_dir, requirements=[requirement], transitive=False
).wait()
wheels = os.listdir(download_dir)
assert len(wheels) == 1, "Expected 1 wheel to be downloaded for {}".format(requirement)
wheel_path = os.path.join(download_dir, wheels[0])
install_dir = os.path.join(td, "install")
yield install_wheel(wheel_path, install_dir=install_dir)
def bdist(self):
get_pip().spawn_build_wheels(distributions=[self._source_dir],
wheel_dir=self._wheel_dir,
interpreter=self._interpreter).wait()
dists = os.listdir(self._wheel_dir)
if len(dists) == 0:
raise self.BuildFailure('No distributions were produced!')
elif len(dists) > 1:
raise self.BuildFailure(
'Ambiguous source distributions found: %s' % (' '.join(dists)))
else:
return os.path.join(self._wheel_dir, dists[0])
def bdist(self):
# type: () -> str
get_pip().spawn_build_wheels(
distributions=[self._source_dir],
wheel_dir=self._wheel_dir,
interpreter=self._interpreter,
).wait()
dists = os.listdir(self._wheel_dir)
if len(dists) == 0:
raise self.BuildFailure("No distributions were produced!")
if len(dists) > 1:
raise self.BuildFailure("Ambiguous source distributions found: %s" % (" ".join(dists)))
return os.path.join(self._wheel_dir, dists[0])
def make_bdist(name='my_project', version='0.0.0', zip_safe=True, interpreter=None, **kwargs):
with built_wheel(name=name,
version=version,
zip_safe=zip_safe,
interpreter=interpreter,
**kwargs) as dist_location:
install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location))
get_pip().spawn_install_wheel(
wheel=dist_location,
install_dir=install_dir,
target=DistributionTarget.for_interpreter(interpreter)
).wait()
yield DistributionHelper.distribution_from_path(install_dir)
def downloaded_sdist(requirement):
# type: (str) -> Iterator[str]
with temporary_dir() as td:
download_dir = os.path.join(td, "download")
get_pip().spawn_download_distributions(
download_dir=download_dir,
requirements=[requirement],
transitive=False,
use_wheel=False,
).wait()
dists = os.listdir(download_dir)
assert len(dists) == 1, "Expected 1 dist to be downloaded for {}.".format(requirement)
sdist = os.path.join(download_dir, dists[0])
assert sdist.endswith((".sdist", ".tar.gz", ".zip"))
yield sdist
def test_get_script_from_distributions(tmpdir):
whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl'
install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path))
get_pip().spawn_install_wheel(wheel=whl_path,
install_dir=install_dir).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert 'aws-cfn-bootstrap' == dist.project_name
dist_script = get_script_from_distributions('cfn-signal', [dist])
assert dist_script.dist is dist
assert os.path.join(install_dir, 'bin/cfn-signal') == dist_script.path
assert dist_script.read_contents().startswith('#!'), (
'Expected a `scripts`-style script w/shebang.')
assert None is get_script_from_distributions('non_existent_script', [dist])
def test_get_script_from_distributions(tmpdir):
# type: (Any) -> None
whl_path = "./tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl"
install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path))
get_pip().spawn_install_wheel(wheel=whl_path,
install_dir=install_dir).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert "aws-cfn-bootstrap" == dist.project_name
dist_script = get_script_from_distributions("cfn-signal", [dist])
assert dist_script.dist is dist
assert os.path.join(install_dir, "bin/cfn-signal") == dist_script.path
assert dist_script.read_contents().startswith(
"#!"), "Expected a `scripts`-style script w/shebang."
assert None is get_script_from_distributions("non_existent_script", [dist])
def _spawn_wheel_build(self, built_wheels_dir, build_request):
build_result = build_request.result(built_wheels_dir)
build_job = get_pip().spawn_build_wheels(
distributions=[build_request.source_path],
wheel_dir=build_result.build_dir,
cache=self._cache,
interpreter=build_request.target.get_interpreter())
return SpawnedJob.wait(job=build_job, result=build_result)
def _spawn_install(self, installed_wheels_dir, install_request):
install_result = install_request.result(installed_wheels_dir)
install_job = get_pip().spawn_install_wheel(
wheel=install_request.wheel_path,
install_dir=install_result.build_chroot,
compile=self._compile,
cache=self._cache,
target=install_request.target)
return SpawnedJob.wait(job=install_job, result=install_result)
def _spawn_wheel_build(self, built_wheels_dir, build_request):
build_result = build_request.result(built_wheels_dir)
build_job = get_pip().spawn_build_wheels(
distributions=[build_request.source_path],
wheel_dir=build_result.build_dir,
cache=self._cache,
indexes=self._indexes,
find_links=self._find_links,
network_configuration=self._network_configuration,
interpreter=build_request.target.get_interpreter())
return SpawnedJob.wait(job=build_job, result=build_result)
def make_bdist(
name="my_project", # type: str
version="0.0.0", # type: str
zip_safe=True, # type: bool
interpreter=None, # type: Optional[PythonInterpreter]
**kwargs # type: Any
):
# type: (...) -> Iterator[Distribution]
with built_wheel(
name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs
) as dist_location:
install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location))
get_pip().spawn_install_wheel(
wheel=dist_location,
install_dir=install_dir,
target=DistributionTarget.for_interpreter(interpreter),
).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None
yield dist
def _spawn_resolve(self, resolved_dists_dir, target):
download_dir = os.path.join(resolved_dists_dir, target.id)
download_job = get_pip().spawn_download_distributions(
download_dir=download_dir,
requirements=self._requirements,
requirement_files=self._requirement_files,
constraint_files=self._constraint_files,
allow_prereleases=self._allow_prereleases,
transitive=self._transitive,
target=target,
indexes=self._indexes,
find_links=self._find_links,
cache=self._cache,
build=self._build,
manylinux=self._manylinux,
use_wheel=self._use_wheel)
return SpawnedJob.wait(job=download_job,
result=ResolveResult(target, download_dir))
def _spawn_download(self, resolved_dists_dir, target):
download_dir = os.path.join(resolved_dists_dir, target.id)
download_job = get_pip().spawn_download_distributions(
download_dir=download_dir,
requirements=self.requirements,
requirement_files=self.requirement_files,
constraint_files=self.constraint_files,
allow_prereleases=self.allow_prereleases,
transitive=self.transitive,
target=target,
indexes=self.indexes,
find_links=self.find_links,
network_configuration=self.network_configuration,
cache=self.cache,
build=self.build,
manylinux=self.manylinux,
use_wheel=self.use_wheel)
return SpawnedJob.wait(job=download_job,
result=DownloadResult(target, download_dir))
def _calculate_tags(
self,
manylinux=None, # type: Optional[str]
):
# type: (...) -> Iterator[tags.Tag]
from pex.jobs import SpawnedJob
from pex.pip import get_pip
def parse_tags(output):
# type: (bytes) -> Iterator[tags.Tag]
count = None # type: Optional[int]
try:
for line in output.decode("utf-8").splitlines():
if count is None:
match = re.match(
r"^Compatible tags: (?P<count>\d+)\s+", line)
if match:
count = int(match.group("count"))
continue
count -= 1
if count < 0:
raise AssertionError(
"Expected {} tags but got more.".format(count))
for tag in tags.parse_tag(line.strip()):
yield tag
finally:
if count != 0:
raise AssertionError(
"Finished with count {}.".format(count))
job = SpawnedJob.stdout(
job=get_pip().spawn_debug(
platform=self.platform,
impl=self.impl,
version=self.version,
abi=self.abi,
manylinux=manylinux,
),
result_func=parse_tags,
)
return job.await_result()
def test_issues_789_demo():
# type: () -> None
tmpdir = safe_mkdtemp()
pex_project_dir = (subprocess.check_output(
["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip())
# 1. Imagine we've pre-resolved the requirements needed in our wheel house.
requirements = [
"ansicolors",
"isort",
"setuptools", # N.B.: isort doesn't declare its setuptools dependency.
]
wheelhouse = os.path.join(tmpdir, "wheelhouse")
get_pip().spawn_download_distributions(download_dir=wheelhouse,
requirements=requirements).wait()
# 2. Also imagine this configuration is passed to a tool (PEX or a wrapper as in this test
# example) via the CLI or other configuration data sources. For example, Pants has a `PythonSetup`
# that combines with BUILD target data to get you this sort of configuration info outside pex.
resolver_settings = dict(
indexes=[], # Turn off pypi.
find_links=[wheelhouse], # Use our wheel house.
build=False, # Use only pre-built wheels.
) # type: Dict[str, Any]
# 3. That same configuration was used to build a standard pex:
resolver_args = []
if len(resolver_settings["find_links"]) == 0:
resolver_args.append("--no-index")
else:
for index in resolver_settings["indexes"]:
resolver_args.extend(["--index", index])
for repo in resolver_settings["find_links"]:
resolver_args.extend(["--find-links", repo])
resolver_args.append(
"--build" if resolver_settings["build"] else "--no-build")
project_code_dir = os.path.join(tmpdir, "project_code_dir")
with safe_open(os.path.join(project_code_dir, "colorized_isort.py"),
"w") as fp:
fp.write(
dedent("""\
import colors
import os
import subprocess
import sys
def run():
env = os.environ.copy()
env.update(PEX_MODULE='isort')
isort_process = subprocess.Popen(
sys.argv,
env=env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout, stderr = isort_process.communicate()
print(colors.green(stdout.decode('utf-8')))
print(colors.red(stderr.decode('utf-8')))
sys.exit(isort_process.returncode)
"""))
colorized_isort_pex = os.path.join(tmpdir, "colorized_isort.pex")
args = [
"--sources-directory",
project_code_dir,
"--entry-point",
"colorized_isort:run",
"--output-file",
colorized_isort_pex,
]
result = run_pex_command(args + resolver_args + requirements)
result.assert_success()
# 4. Now the tool builds a "dehydrated" PEX using the standard pex + resolve settings as the
# template.
ptex_cache = os.path.join(tmpdir, ".ptex")
colorized_isort_pex_info = PexInfo.from_pex(colorized_isort_pex)
colorized_isort_pex_info.pex_root = ptex_cache
# Force the standard pex to extract its code. An external tool like Pants would already know the
# orignal source code file paths, but we need to discover here.
colorized_isort_pex_code_dir = os.path.join(
colorized_isort_pex_info.zip_unsafe_cache,
colorized_isort_pex_info.code_hash)
env = os.environ.copy()
env.update(PEX_ROOT=ptex_cache, PEX_INTERPRETER="1", PEX_FORCE_LOCAL="1")
subprocess.check_call([colorized_isort_pex, "-c", ""], env=env)
colorized_isort_ptex_code_dir = os.path.join(
tmpdir, "colorized_isort_ptex_code_dir")
safe_mkdir(colorized_isort_ptex_code_dir)
code = []
for root, dirs, files in os.walk(colorized_isort_pex_code_dir):
rel_root = os.path.relpath(root, colorized_isort_pex_code_dir)
for f in files:
# Don't ship compiled python from the code extract above, the target interpreter will not
# match ours in general.
if f.endswith(".pyc"):
continue
rel_path = os.path.normpath(os.path.join(rel_root, f))
# The root __main__.py is special for any zipapp including pex, let it write its own
# __main__.py bootstrap. Similarly. PEX-INFO is special to pex and we want the PEX-INFO for
# The ptex pex, not the pex being ptexed.
if rel_path in ("__main__.py", PexInfo.PATH):
continue
os.symlink(os.path.join(root, f),
os.path.join(colorized_isort_ptex_code_dir, rel_path))
code.append(rel_path)
ptex_code_dir = os.path.join(tmpdir, "ptex_code_dir")
ptex_info = dict(code=code, resolver_settings=resolver_settings)
with safe_open(os.path.join(ptex_code_dir, "PTEX-INFO"), "w") as fp:
json.dump(ptex_info, fp)
with safe_open(os.path.join(ptex_code_dir, "IPEX-INFO"), "w") as fp:
fp.write(colorized_isort_pex_info.dump())
with safe_open(os.path.join(ptex_code_dir, "ptex.py"), "w") as fp:
fp.write(
dedent("""\
import json
import os
import sys
from pex import resolver
from pex.common import open_zip
from pex.pex_builder import PEXBuilder
from pex.pex_info import PexInfo
from pex.util import CacheHelper
from pex.variables import ENV
self = sys.argv[0]
ipex_file = '{}.ipex'.format(os.path.splitext(self)[0])
if not os.path.isfile(ipex_file):
print('Hydrating {} to {}'.format(self, ipex_file))
ptex_pex_info = PexInfo.from_pex(self)
code_root = os.path.join(ptex_pex_info.zip_unsafe_cache, ptex_pex_info.code_hash)
with open_zip(self) as zf:
# Populate the pex with the pinned requirements and distribution names & hashes.
ipex_info = PexInfo.from_json(zf.read('IPEX-INFO'))
ipex_builder = PEXBuilder(pex_info=ipex_info)
# Populate the pex with the needed code.
ptex_info = json.loads(zf.read('PTEX-INFO').decode('utf-8'))
for path in ptex_info['code']:
ipex_builder.add_source(os.path.join(code_root, path), path)
# Perform a fully pinned intransitive resolve to hydrate the install cache (not the
# pex!).
resolver_settings = ptex_info['resolver_settings']
resolved_distributions = resolver.resolve(
requirements=[str(req) for req in ipex_info.requirements],
cache=ipex_info.pex_root,
transitive=False,
**resolver_settings
)
ipex_builder.build(ipex_file)
os.execv(ipex_file, [ipex_file] + sys.argv[1:])
"""))
colorized_isort_ptex = os.path.join(tmpdir, "colorized_isort.ptex")
result = run_pex_command([
"--not-zip-safe",
"--always-write-cache",
"--pex-root",
ptex_cache,
pex_project_dir, # type: ignore[list-item] # This is unicode in Py2, whereas everthing else is bytes. That's fine.
"--sources-directory",
ptex_code_dir,
"--sources-directory",
colorized_isort_ptex_code_dir,
"--entry-point",
"ptex",
"--output-file",
colorized_isort_ptex,
])
result.assert_success()
subprocess.check_call([colorized_isort_ptex, "--version"])
with pytest.raises(CalledProcessError):
subprocess.check_call([colorized_isort_ptex, "--not-a-flag"])
safe_rmtree(ptex_cache)
# The dehydrated pex now fails since it lost its hydration from the cache.
with pytest.raises(CalledProcessError):
subprocess.check_call([colorized_isort_ptex, "--version"])
