def add_from_requirements_pex(self, pex):
"""Add requirements from an existing pex.
:param pex: The path to an existing .pex file or unzipped pex directory.
"""
self._ensure_unfrozen("Adding from pex")
pex_info = PexInfo.from_pex(pex)
def add(location, dname, expected_dhash):
dhash = self._add_dist_dir(location, dname)
if dhash != expected_dhash:
raise self.InvalidDistribution(
"Distribution {} at {} had hash {}, expected {}".format(
dname, location, dhash, expected_dhash))
self._pex_info.add_distribution(dname, dhash)
if os.path.isfile(pex):
with open_zip(pex) as zf:
for dist_name, dist_hash in pex_info.distributions.items():
internal_dist_path = "/".join(
[pex_info.internal_cache, dist_name])
cached_location = os.path.join(pex_info.install_cache,
dist_hash, dist_name)
CacheHelper.cache_distribution(zf, internal_dist_path,
cached_location)
add(cached_location, dist_name, dist_hash)
else:
for dist_name, dist_hash in pex_info.distributions.items():
add(os.path.join(pex, pex_info.internal_cache, dist_name),
dist_name, dist_hash)
for req in pex_info.requirements:
self._pex_info.add_requirement(req)
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def _write_zipped_internal_cache(cls, zf, pex_info):
cached_distributions = []
for distribution_name, dist_digest in pex_info.distributions.items():
internal_dist_path = "/".join([pex_info.internal_cache, distribution_name])
cached_location = os.path.join(pex_info.install_cache, dist_digest, distribution_name)
dist = CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)
cached_distributions.append(dist)
return cached_distributions
