def test_dir_hash():
# type: () -> None
with temporary_dir() as tmp_dir:
safe_mkdir(os.path.join(tmp_dir, "a", "b"))
with safe_open(os.path.join(tmp_dir, "c", "d", "e.py"), "w") as fp:
fp.write("contents1")
with safe_open(os.path.join(tmp_dir, "f.py"), "w") as fp:
fp.write("contents2")
hash1 = CacheHelper.dir_hash(tmp_dir)
os.rename(os.path.join(tmp_dir, "c"),
os.path.join(tmp_dir, "c-renamed"))
assert hash1 != CacheHelper.dir_hash(tmp_dir)
os.rename(os.path.join(tmp_dir, "c-renamed"),
os.path.join(tmp_dir, "c"))
assert hash1 == CacheHelper.dir_hash(tmp_dir)
touch(os.path.join(tmp_dir, "c", "d", "e.pyc"))
assert hash1 == CacheHelper.dir_hash(tmp_dir)
touch(os.path.join(tmp_dir, "c", "d", "e.pyc.123456789"))
assert hash1 == CacheHelper.dir_hash(tmp_dir)
pycache_dir = os.path.join(tmp_dir, "__pycache__")
safe_mkdir(pycache_dir)
touch(os.path.join(pycache_dir, "f.pyc"))
assert hash1 == CacheHelper.dir_hash(tmp_dir)
touch(os.path.join(pycache_dir, "f.pyc.123456789"))
assert hash1 == CacheHelper.dir_hash(tmp_dir)
touch(os.path.join(pycache_dir, "f.py"))
assert hash1 == CacheHelper.dir_hash(
tmp_dir
), "All content under __pycache__ directories should be ignored."
def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache, pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name, name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache,
pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name,
name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def finalize_install(self, install_requests):
self.atomic_dir.finalize()
# The install_chroot is keyed by the hash of the wheel file (zip) we installed. Here we add a
# key by the hash of the exploded wheel dir (the install_chroot). This latter key is used by
# zipped PEXes at runtime to explode their wheel chroots to the filesystem. By adding the key
# here we short-circuit the explode process for PEXes created and run on the same machine.
#
# From a clean cache after building a simple pex this looks like:
# $ rm -rf ~/.pex
# $ python -mpex -c pex -o /tmp/pex.pex .
# $ tree -L 4 ~/.pex/
# /home/jsirois/.pex/
# ├── built_wheels
# │ └── 1003685de2c3604dc6daab9540a66201c1d1f718
# │ └── cp-38-cp38
# │ └── pex-2.0.2-py2.py3-none-any.whl
# └── installed_wheels
# ├── 2a594cef34d2e9109bad847358d57ac4615f81f4
# │ └── pex-2.0.2-py2.py3-none-any.whl
# │ ├── bin
# │ ├── pex
# │ └── pex-2.0.2.dist-info
# └── ae13cba3a8e50262f4d730699a11a5b79536e3e1
# └── pex-2.0.2-py2.py3-none-any.whl -> /home/jsirois/.pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl # noqa
#
# 11 directories, 1 file
#
# And we see in the created pex, the runtime key that the layout above satisfies:
# $ unzip -qc /tmp/pex.pex PEX-INFO | jq .distributions
# {
# "pex-2.0.2-py2.py3-none-any.whl": "ae13cba3a8e50262f4d730699a11a5b79536e3e1"
# }
#
# When the pex is run, the runtime key is followed to the build time key, avoiding re-unpacking
# the wheel:
# $ PEX_VERBOSE=1 /tmp/pex.pex --version
# pex: Found site-library: /usr/lib/python3.8/site-packages
# pex: Tainted path element: /usr/lib/python3.8/site-packages
# pex: Scrubbing from user site: /home/jsirois/.local/lib/python3.8/site-packages
# pex: Scrubbing from site-packages: /usr/lib/python3.8/site-packages
# pex: Activating PEX virtual environment from /tmp/pex.pex: 9.1ms
# pex: Bootstrap complete, performing final sys.path modifications...
# pex: PYTHONPATH contains:
# pex: /tmp/pex.pex
# pex: * /usr/lib/python38.zip
# pex: /usr/lib/python3.8
# pex: /usr/lib/python3.8/lib-dynload
# pex: /home/jsirois/.pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl # noqa
# pex: * /tmp/pex.pex/.bootstrap
# pex: * - paths that do not exist or will be imported via zipimport
# pex.pex 2.0.2
#
wheel_dir_hash = CacheHelper.dir_hash(self.install_chroot)
runtime_key_dir = os.path.join(self.installation_root, wheel_dir_hash)
with atomic_directory(runtime_key_dir) as work_dir:
if work_dir:
os.symlink(self.install_chroot, os.path.join(work_dir, self.request.wheel_file))
return self._iter_requirements_requests(install_requests)
def _add_dist_dir(self, path, dist_name):
for root, _, files in os.walk(path):
for f in files:
filename = os.path.join(root, f)
relpath = os.path.relpath(filename, path)
target = os.path.join(self._pex_info.internal_cache, dist_name, relpath)
self._copy_or_link(filename, target)
return CacheHelper.dir_hash(path)
def _add_dist_dir(self, path, dist_name):
for root, _, files in os.walk(path):
for f in files:
filename = os.path.join(root, f)
relpath = os.path.relpath(filename, path)
target = os.path.join(self._pex_info.internal_cache, dist_name, relpath)
self._copy_or_link(filename, target)
return CacheHelper.dir_hash(path)
def isolated():
"""Returns a chroot for third_party isolated from the ``sys.path``.
PEX will typically be installed in site-packages flat alongside many other distributions; as such,
adding the location of the pex distribution to the ``sys.path`` will typically expose many other
distributions. An isolated chroot can be used as a ``sys.path`` entry to effect only the exposure
of pex.
:return: An isolation result.
:rtype: :class:`IsolationResult`
"""
global _ISOLATED
if _ISOLATED is None:
from pex import vendor
from pex.common import atomic_directory
from pex.util import CacheHelper
from pex.variables import ENV
from pex.third_party.pkg_resources import resource_isdir, resource_listdir, resource_stream
module = "pex"
# TODO(John Sirois): Unify with `pex.util.DistributionHelper.access_zipped_assets`.
def recursive_copy(srcdir, dstdir):
os.mkdir(dstdir)
for entry_name in resource_listdir(module, srcdir):
if not entry_name:
# The `resource_listdir` function returns a '' entry name for the directory
# entry itself if it is either present on the filesystem or present as an
# explicit zip entry. Since we only care about files and subdirectories at this
# point, skip these entries.
continue
# NB: Resource path components are always separated by /, on all systems.
src_entry = "{}/{}".format(
srcdir, entry_name) if srcdir else entry_name
dst_entry = os.path.join(dstdir, entry_name)
if resource_isdir(module, src_entry):
recursive_copy(src_entry, dst_entry)
elif not entry_name.endswith(".pyc"):
with open(dst_entry, "wb") as fp:
with closing(resource_stream(module,
src_entry)) as resource:
shutil.copyfileobj(resource, fp)
pex_path = os.path.join(vendor.VendorSpec.ROOT, "pex")
with _tracer().timed("Hashing pex"):
dir_hash = CacheHelper.dir_hash(pex_path)
isolated_dir = os.path.join(ENV.PEX_ROOT, "isolated", dir_hash)
with _tracer().timed("Isolating pex"):
with atomic_directory(isolated_dir, exclusive=True) as chroot:
if chroot:
with _tracer().timed(
"Extracting pex to {}".format(isolated_dir)):
recursive_copy("", os.path.join(chroot, "pex"))
_ISOLATED = IsolationResult(pex_hash=dir_hash,
chroot_path=isolated_dir)
return _ISOLATED
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with open_zip(tf.name, 'r') as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def _add_dist_dir(self, path, dist_name):
target_dir = os.path.join(self._pex_info.internal_cache, dist_name)
if self._copy_mode == CopyMode.SYMLINK:
self._copy_or_link(path, target_dir)
else:
for root, _, files in os.walk(path):
for f in files:
filename = os.path.join(root, f)
relpath = os.path.relpath(filename, path)
target = os.path.join(target_dir, relpath)
self._copy_or_link(filename, target)
return CacheHelper.dir_hash(path)
def isolated():
"""Returns a chroot for third_party isolated from the ``sys.path``.
PEX will typically be installed in site-packages flat alongside many other distributions; as such,
adding the location of the pex distribution to the ``sys.path`` will typically expose many other
distributions. An isolated chroot can be used as a ``sys.path`` entry to effect only the exposure
of pex.
:return: An isolation result.
:rtype: :class:`IsolationResult`
"""
global _ISOLATED
if _ISOLATED is None:
from pex import vendor
from pex.common import atomic_directory
from pex.util import CacheHelper
from pex.variables import ENV
from pex.third_party.pkg_resources import resource_isdir, resource_listdir, resource_stream
module = 'pex'
def recursive_copy(srcdir, dstdir):
os.mkdir(dstdir)
for entry_name in resource_listdir(module, srcdir):
# NB: Resource path components are always separated by /, on all systems.
src_entry = '{}/{}'.format(
srcdir, entry_name) if srcdir else entry_name
dst_entry = os.path.join(dstdir, entry_name)
if resource_isdir(module, src_entry):
recursive_copy(src_entry, dst_entry)
elif not entry_name.endswith('.pyc'):
with open(dst_entry, 'wb') as fp:
shutil.copyfileobj(resource_stream(module, src_entry),
fp)
pex_path = os.path.join(vendor.VendorSpec.ROOT, 'pex')
with _tracer().timed('Hashing pex'):
dir_hash = CacheHelper.dir_hash(pex_path)
isolated_dir = os.path.join(ENV.PEX_ROOT, 'isolated', dir_hash)
with _tracer().timed('Isolating pex'):
with atomic_directory(isolated_dir) as chroot:
if chroot:
with _tracer().timed(
'Extracting pex to {}'.format(isolated_dir)):
recursive_copy('', os.path.join(chroot, 'pex'))
_ISOLATED = IsolationResult(pex_hash=dir_hash,
chroot_path=isolated_dir)
return _ISOLATED
def isolated():
"""Returns a chroot for third_party isolated from the ``sys.path``.
PEX will typically be installed in site-packages flat alongside many other distributions; as such,
adding the location of the pex distribution to the ``sys.path`` will typically expose many other
distributions. An isolated chroot can be used as a ``sys.path`` entry to effect only the exposure
of pex.
:return: The path of the chroot.
:rtype: str
"""
global _ISOLATED
if _ISOLATED is None:
from pex import vendor
from pex.common import atomic_directory, safe_copy
from pex.util import CacheHelper
from pex.variables import ENV
pex_path = os.path.join(vendor.VendorSpec.ROOT, 'pex')
with _tracer().timed('Isolating pex'):
isolated_dir = os.path.join(ENV.PEX_ROOT, 'isolated',
CacheHelper.dir_hash(pex_path))
with atomic_directory(isolated_dir) as chroot:
if chroot:
with _tracer().timed(
'Extracting pex to {}'.format(isolated_dir)):
pex_path = os.path.join(vendor.VendorSpec.ROOT, 'pex')
for root, dirs, files in os.walk(pex_path):
relroot = os.path.relpath(root, pex_path)
for d in dirs:
os.makedirs(
os.path.join(chroot, 'pex', relroot, d))
for f in files:
if not f.endswith('.pyc'):
safe_copy(
os.path.join(root, f),
os.path.join(chroot, 'pex', relroot,
f))
_ISOLATED = isolated_dir
return _ISOLATED
