def _add_dist_zip(self, path, dist_name): # We need to distinguish between wheels and other zips. Most of the time, # when we have a zip, it contains its contents in an importable form. # But wheels don't have to be importable, so we need to force them # into an importable shape. We can do that by installing it into its own # wheel dir. if dist_name.endswith("whl"): from pex.third_party.wheel.install import WheelFile tmp = safe_mkdtemp() whltmp = os.path.join(tmp, dist_name) os.mkdir(whltmp) wf = WheelFile(path) wf.install(overrides=self._get_installer_paths(whltmp), force=True) for root, _, files in os.walk(whltmp): pruned_dir = os.path.relpath(root, tmp) for f in files: fullpath = os.path.join(root, f) target = os.path.join(self._pex_info.internal_cache, pruned_dir, f) self._copy_or_link(fullpath, target) return CacheHelper.dir_hash(whltmp) with open_zip(path) as zf: for name in zf.namelist(): if name.endswith('/'): continue target = os.path.join(self._pex_info.internal_cache, dist_name, name) self._chroot.write(zf.read(name), target) return CacheHelper.zip_hash(zf)
def write_zipped_internal_cache(cls, pex, pex_info): prefix_length = len(pex_info.internal_cache) + 1 existing_cached_distributions = [] newly_cached_distributions = [] zip_safe_distributions = [] with open_zip(pex) as zf: # Distribution names are the first element after ".deps/" and before the next "/" distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0] for filename in zf.namelist() if filename.startswith(pex_info.internal_cache)))) # Create Distribution objects from these, and possibly write to disk if necessary. for distribution_name in distribution_names: internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name]) # First check if this is already cached dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash( zf, internal_dist_path) cached_location = os.path.join(pex_info.install_cache, '%s.%s' % ( distribution_name, dist_digest)) if os.path.exists(cached_location): dist = DistributionHelper.distribution_from_path(cached_location) if dist is not None: existing_cached_distributions.append(dist) continue else: dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path)) if dist is not None: if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache: zip_safe_distributions.append(dist) continue with TRACER.timed('Caching %s' % dist): newly_cached_distributions.append( CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)) return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def test_hash_consistency(): for reverse in (False, True): with temporary_content(CONTENT) as td: dir_hash = CacheHelper.dir_hash(td) with named_temporary_file() as tf: write_zipfile(td, tf.name, reverse=reverse) with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf: zip_hash = CacheHelper.zip_hash(zf) assert zip_hash == dir_hash assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash_consistency(): for reverse in (False, True): with temporary_content(CONTENT) as td: dir_hash = CacheHelper.dir_hash(td) with named_temporary_file() as tf: write_zipfile(td, tf.name, reverse=reverse) with open_zip(tf.name, 'r') as zf: zip_hash = CacheHelper.zip_hash(zf) assert zip_hash == dir_hash assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def isolated(): """Returns a chroot for third_party isolated from the ``sys.path``. PEX will typically be installed in site-packages flat alongside many other distributions; as such, adding the location of the pex distribution to the ``sys.path`` will typically expose many other distributions. An isolated chroot can be used as a ``sys.path`` entry to effect only the exposure of pex. :return: An isolation result. :rtype: :class:`IsolationResult` """ global _ISOLATED if _ISOLATED is None: from pex import vendor from pex.common import atomic_directory from pex.util import CacheHelper from pex.variables import ENV from pex.third_party.pkg_resources import resource_isdir, resource_listdir, resource_stream module = "pex" # TODO(John Sirois): Unify with `pex.util.DistributionHelper.access_zipped_assets`. def recursive_copy(srcdir, dstdir): os.mkdir(dstdir) for entry_name in resource_listdir(module, srcdir): if not entry_name: # The `resource_listdir` function returns a '' entry name for the directory # entry itself if it is either present on the filesystem or present as an # explicit zip entry. Since we only care about files and subdirectories at this # point, skip these entries. continue # NB: Resource path components are always separated by /, on all systems. src_entry = "{}/{}".format(srcdir, entry_name) if srcdir else entry_name dst_entry = os.path.join(dstdir, entry_name) if resource_isdir(module, src_entry): recursive_copy(src_entry, dst_entry) elif not entry_name.endswith(".pyc"): with open(dst_entry, "wb") as fp: with closing(resource_stream(module, src_entry)) as resource: shutil.copyfileobj(resource, fp) pex_path = os.path.join(vendor.VendorSpec.ROOT, "pex") with _tracer().timed("Hashing pex"): if os.path.isdir(pex_path): dir_hash = CacheHelper.dir_hash(pex_path) else: pex_zip = os.path.abspath(sys.argv[0]) assert zipfile.is_zipfile(pex_zip) and pex_zip == os.path.commonprefix( (pex_zip, pex_path) ), ( "Expected the `pex` module to be available via an installed distribution or " "else via a PEX zipfile present as argv0. Loaded the `pex` module from {} and " "argv0 is {}.".format(pex_path, sys.argv[0]) ) dir_hash = CacheHelper.zip_hash(pex_zip, os.path.relpath(pex_path, pex_zip)) isolated_dir = os.path.join(ENV.PEX_ROOT, "isolated", dir_hash) with _tracer().timed("Isolating pex"): with atomic_directory(isolated_dir, exclusive=True) as chroot: if chroot: with _tracer().timed("Extracting pex to {}".format(isolated_dir)): recursive_copy("", os.path.join(chroot, "pex")) _ISOLATED = IsolationResult(pex_hash=dir_hash, chroot_path=isolated_dir) return _ISOLATED