def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache, pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name, name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache,
pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name,
name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with open_zip(tf.name, 'r') as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def isolated():
"""Returns a chroot for third_party isolated from the ``sys.path``.
PEX will typically be installed in site-packages flat alongside many other distributions; as such,
adding the location of the pex distribution to the ``sys.path`` will typically expose many other
distributions. An isolated chroot can be used as a ``sys.path`` entry to effect only the exposure
of pex.
:return: An isolation result.
:rtype: :class:`IsolationResult`
"""
global _ISOLATED
if _ISOLATED is None:
from pex import vendor
from pex.common import atomic_directory
from pex.util import CacheHelper
from pex.variables import ENV
from pex.third_party.pkg_resources import resource_isdir, resource_listdir, resource_stream
module = "pex"
# TODO(John Sirois): Unify with `pex.util.DistributionHelper.access_zipped_assets`.
def recursive_copy(srcdir, dstdir):
os.mkdir(dstdir)
for entry_name in resource_listdir(module, srcdir):
if not entry_name:
# The `resource_listdir` function returns a '' entry name for the directory
# entry itself if it is either present on the filesystem or present as an
# explicit zip entry. Since we only care about files and subdirectories at this
# point, skip these entries.
continue
# NB: Resource path components are always separated by /, on all systems.
src_entry = "{}/{}".format(srcdir, entry_name) if srcdir else entry_name
dst_entry = os.path.join(dstdir, entry_name)
if resource_isdir(module, src_entry):
recursive_copy(src_entry, dst_entry)
elif not entry_name.endswith(".pyc"):
with open(dst_entry, "wb") as fp:
with closing(resource_stream(module, src_entry)) as resource:
shutil.copyfileobj(resource, fp)
pex_path = os.path.join(vendor.VendorSpec.ROOT, "pex")
with _tracer().timed("Hashing pex"):
if os.path.isdir(pex_path):
dir_hash = CacheHelper.dir_hash(pex_path)
else:
pex_zip = os.path.abspath(sys.argv[0])
assert zipfile.is_zipfile(pex_zip) and pex_zip == os.path.commonprefix(
(pex_zip, pex_path)
), (
"Expected the `pex` module to be available via an installed distribution or "
"else via a PEX zipfile present as argv0. Loaded the `pex` module from {} and "
"argv0 is {}.".format(pex_path, sys.argv[0])
)
dir_hash = CacheHelper.zip_hash(pex_zip, os.path.relpath(pex_path, pex_zip))
isolated_dir = os.path.join(ENV.PEX_ROOT, "isolated", dir_hash)
with _tracer().timed("Isolating pex"):
with atomic_directory(isolated_dir, exclusive=True) as chroot:
if chroot:
with _tracer().timed("Extracting pex to {}".format(isolated_dir)):
recursive_copy("", os.path.join(chroot, "pex"))
_ISOLATED = IsolationResult(pex_hash=dir_hash, chroot_path=isolated_dir)
return _ISOLATED
