def _add_dist_zip(self, path, dist_name): # We need to distinguish between wheels and other zips. Most of the time, # when we have a zip, it contains its contents in an importable form. # But wheels don't have to be importable, so we need to force them # into an importable shape. We can do that by installing it into its own # wheel dir. if dist_name.endswith("whl"): from pex.third_party.wheel.install import WheelFile tmp = safe_mkdtemp() whltmp = os.path.join(tmp, dist_name) os.mkdir(whltmp) wf = WheelFile(path) wf.install(overrides=self._get_installer_paths(whltmp), force=True) for root, _, files in os.walk(whltmp): pruned_dir = os.path.relpath(root, tmp) for f in files: fullpath = os.path.join(root, f) target = os.path.join(self._pex_info.internal_cache, pruned_dir, f) self._copy_or_link(fullpath, target) return CacheHelper.dir_hash(whltmp) with open_zip(path) as zf: for name in zf.namelist(): if name.endswith('/'): continue target = os.path.join(self._pex_info.internal_cache, dist_name, name) self._chroot.write(zf.read(name), target) return CacheHelper.zip_hash(zf)
def write_zipped_internal_cache(cls, pex, pex_info): prefix_length = len(pex_info.internal_cache) + 1 existing_cached_distributions = [] newly_cached_distributions = [] zip_safe_distributions = [] with open_zip(pex) as zf: # Distribution names are the first element after ".deps/" and before the next "/" distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0] for filename in zf.namelist() if filename.startswith(pex_info.internal_cache)))) # Create Distribution objects from these, and possibly write to disk if necessary. for distribution_name in distribution_names: internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name]) # First check if this is already cached dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash( zf, internal_dist_path) cached_location = os.path.join(pex_info.install_cache, '%s.%s' % ( distribution_name, dist_digest)) if os.path.exists(cached_location): dist = DistributionHelper.distribution_from_path(cached_location) if dist is not None: existing_cached_distributions.append(dist) continue else: dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path)) if dist is not None: if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache: zip_safe_distributions.append(dist) continue with TRACER.timed('Caching %s' % dist): newly_cached_distributions.append( CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)) return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def write_zipped_internal_cache(cls, pex, pex_info): prefix_length = len(pex_info.internal_cache) + 1 existing_cached_distributions = [] newly_cached_distributions = [] zip_safe_distributions = [] with open_zip(pex) as zf: # Distribution names are the first element after ".deps/" and before the next "/" distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0] for filename in zf.namelist() if filename.startswith(pex_info.internal_cache)))) # Create Distribution objects from these, and possibly write to disk if necessary. for distribution_name in distribution_names: internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name]) # First check if this is already cached dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash( zf, internal_dist_path) cached_location = os.path.join(pex_info.install_cache, '%s.%s' % ( distribution_name, dist_digest)) if os.path.exists(cached_location): dist = DistributionHelper.distribution_from_path(cached_location) if dist is not None: existing_cached_distributions.append(dist) continue else: dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path)) if dist is not None: if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache: zip_safe_distributions.append(dist) continue with TRACER.timed('Caching %s' % dist): newly_cached_distributions.append( CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)) return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def _add_dist_zip(self, path, dist_name): # We need to distinguish between wheels and other zips. Most of the time, # when we have a zip, it contains its contents in an importable form. # But wheels don't have to be importable, so we need to force them # into an importable shape. We can do that by installing it into its own # wheel dir. if dist_name.endswith("whl"): from pex.third_party.wheel.install import WheelFile tmp = safe_mkdtemp() whltmp = os.path.join(tmp, dist_name) os.mkdir(whltmp) wf = WheelFile(path) wf.install(overrides=self._get_installer_paths(whltmp), force=True) for root, _, files in os.walk(whltmp): pruned_dir = os.path.relpath(root, tmp) for f in files: fullpath = os.path.join(root, f) target = os.path.join(self._pex_info.internal_cache, pruned_dir, f) self._copy_or_link(fullpath, target) return CacheHelper.dir_hash(whltmp) with open_zip(path) as zf: for name in zf.namelist(): if name.endswith('/'): continue target = os.path.join(self._pex_info.internal_cache, dist_name, name) self._chroot.write(zf.read(name), target) return CacheHelper.zip_hash(zf)
def get_dep_dist_names_from_pex(pex_path, match_prefix=""): # type: (str, str) -> Set[str] """Given an on-disk pex, extract all of the unique first-level paths under `.deps`.""" with open_zip(pex_path) as pex_zip: dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith(".deps/")) return set(item for item in dep_gen if item.startswith(match_prefix))
def test_pex_builder(): # test w/ and w/o zipfile dists with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1): pb = write_pex(td, exe_main, dists=[p1]) success_txt = os.path.join(td, 'success.txt') PEX(td, interpreter=pb.interpreter).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success' # test w/ and w/o zipfile dists with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as ( td1, td2, p1): target_egg_dir = os.path.join(td2, os.path.basename(p1.location)) safe_mkdir(target_egg_dir) with open_zip(p1.location, 'r') as zf: zf.extractall(target_egg_dir) p1 = DistributionHelper.distribution_from_path(target_egg_dir) pb = write_pex(td1, exe_main, dists=[p1]) success_txt = os.path.join(td1, 'success.txt') PEX(td1, interpreter=pb.interpreter).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success'
def _parse_sdist_package_info(sdist_path): # type: (str) -> Optional[Message] sdist_filename = _strip_sdist_path(sdist_path) if sdist_filename is None: return None pkg_info_path = os.path.join(sdist_filename, Distribution.PKG_INFO) if zipfile.is_zipfile(sdist_path): with open_zip(sdist_path) as zip: try: return _parse_message(zip.read(pkg_info_path).decode("utf-8")) except KeyError as e: pex_warnings.warn( "Source distribution {} did not have the expected metadata file {}: {}" .format(sdist_path, pkg_info_path, e)) return None if tarfile.is_tarfile(sdist_path): with tarfile.open(sdist_path) as tf: try: pkg_info = tf.extractfile(pkg_info_path) if pkg_info is None: # N.B.: `extractfile` returns None for directories and special files. return None with closing(pkg_info) as fp: return _parse_message(fp.read().decode("utf-8")) except KeyError as e: pex_warnings.warn( "Source distribution {} did not have the expected metadata file {}: {}" .format(sdist_path, pkg_info_path, e)) return None return None
def test_unwriteable_contents(): my_app_setup_py = dedent(""" from setuptools import setup setup( name='my_app', version='0.0.0', zip_safe=True, packages=['my_app'], include_package_data=True, package_data={'my_app': ['unwriteable.so']}, ) """) UNWRITEABLE_PERMS = 0o400 with temporary_content({'setup.py': my_app_setup_py, 'my_app/__init__.py': '', 'my_app/unwriteable.so': 'so contents'}, perms=UNWRITEABLE_PERMS) as my_app_project_dir: my_app_whl = WheelBuilder(my_app_project_dir).bdist() with make_project(name='uses_my_app', install_reqs=['my_app']) as uses_my_app_project_dir: pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl)) with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex: with open_zip(uses_my_app_pex) as zf: unwriteable_sos = [path for path in zf.namelist() if path.endswith('my_app/unwriteable.so')] assert 1 == len(unwriteable_sos) unwriteable_so = unwriteable_sos.pop() zf.extract(unwriteable_so, path=uses_my_app_project_dir) extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so) with open(extract_dest) as fp: assert 'so contents' == fp.read()
def test_pex_builder(): # test w/ and w/o zipfile dists with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1): pb = write_pex(td, exe_main, dists=[p1]) success_txt = os.path.join(td, 'success.txt') PEX(td, interpreter=pb.interpreter).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success' # test w/ and w/o zipfile dists with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as ( td1, td2, p1): target_egg_dir = os.path.join(td2, os.path.basename(p1.location)) safe_mkdir(target_egg_dir) with open_zip(p1.location, 'r') as zf: zf.extractall(target_egg_dir) p1 = DistributionHelper.distribution_from_path(target_egg_dir) pb = write_pex(td1, exe_main, dists=[p1]) success_txt = os.path.join(td1, 'success.txt') PEX(td1, interpreter=pb.interpreter).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success'
def assert_dist_cache(zip_safe): # type: (bool) -> None with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file, ): pb.info.pex_root = pex_root pb.build(pex_file) with open_zip(pex_file) as zf: dists = PEXEnvironment._write_zipped_internal_cache( zf=zf, pex_info=pb.info) assert len(dists) == 1 original_location = normalize(dists[0].location) assert original_location.startswith( normalize(pb.info.install_cache)) # Call a second time to validate idempotence of caching. dists = PEXEnvironment._write_zipped_internal_cache(zf=None, pex_info=pb.info) assert len(dists) == 1 assert normalize(dists[0].location) == original_location
def seed_cache( options, # type: Namespace pex, # type: PEX ): # type: (...) -> Iterable[str] pex_path = pex.path() with TRACER.timed("Seeding local caches for {}".format(pex_path)): if options.unzip: unzip_dir = pex.pex_info().unzip_dir if unzip_dir is None: raise AssertionError( "Expected PEX-INFO for {} to have the components of an unzip directory" .format(pex_path)) with atomic_directory(unzip_dir, exclusive=True) as chroot: if chroot: with TRACER.timed("Extracting {}".format(pex_path)): with open_zip(options.pex_name) as pex_zip: pex_zip.extractall(chroot) return [pex.interpreter.binary, unzip_dir] elif options.venv: with TRACER.timed("Creating venv from {}".format(pex_path)): venv_pex = ensure_venv(pex) return [venv_pex] else: with TRACER.timed( "Extracting code and distributions for {}".format( pex_path)): pex.activate() return [os.path.abspath(options.pex_name)]
def add_from_requirements_pex(self, pex): """Add requirements from an existing pex. :param pex: The path to an existing .pex file or unzipped pex directory. """ self._ensure_unfrozen("Adding from pex") pex_info = PexInfo.from_pex(pex) def add(location, dname, expected_dhash): dhash = self._add_dist_dir(location, dname) if dhash != expected_dhash: raise self.InvalidDistribution( "Distribution {} at {} had hash {}, expected {}".format( dname, location, dhash, expected_dhash)) self._pex_info.add_distribution(dname, dhash) if os.path.isfile(pex): with open_zip(pex) as zf: for dist_name, dist_hash in pex_info.distributions.items(): internal_dist_path = "/".join( [pex_info.internal_cache, dist_name]) cached_location = os.path.join(pex_info.install_cache, dist_hash, dist_name) CacheHelper.cache_distribution(zf, internal_dist_path, cached_location) add(cached_location, dist_name, dist_hash) else: for dist_name, dist_hash in pex_info.distributions.items(): add(os.path.join(pex, pex_info.internal_cache, dist_name), dist_name, dist_hash) for req in pex_info.requirements: self._pex_info.add_requirement(req)
def from_pex(cls, pex): if os.path.isfile(pex): with open_zip(pex) as zf: pex_info = zf.read(cls.PATH) else: with open(os.path.join(pex, cls.PATH)) as fp: pex_info = fp.read() return cls.from_json(pex_info)
def from_pex(cls, pex): if os.path.isfile(pex): with open_zip(pex) as zf: pex_info = zf.read(cls.PATH) else: with open(os.path.join(pex, cls.PATH)) as fp: pex_info = fp.read() return cls.from_json(pex_info)
def _parse_wheel_package_info(wheel_path): # type: (str) -> Optional[Message] if not wheel_path.endswith(".whl") or not zipfile.is_zipfile(wheel_path): return None project_name, version, _ = os.path.basename(wheel_path).split("-", 2) dist_info_dir = "{}-{}.dist-info".format(project_name, version) with open_zip(wheel_path) as whl: with whl.open(os.path.join(dist_info_dir, DistInfoDistribution.PKG_INFO)) as fp: return _parse_message(fp.read())
def test_hash_consistency(): for reverse in (False, True): with temporary_content(CONTENT) as td: dir_hash = CacheHelper.dir_hash(td) with named_temporary_file() as tf: write_zipfile(td, tf.name, reverse=reverse) with open_zip(tf.name, 'r') as zf: zip_hash = CacheHelper.zip_hash(zf) assert zip_hash == dir_hash assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash_consistency(): for reverse in (False, True): with temporary_content(CONTENT) as td: dir_hash = CacheHelper.dir_hash(td) with named_temporary_file() as tf: write_zipfile(td, tf.name, reverse=reverse) with open_zip(tf.name, 'r') as zf: zip_hash = CacheHelper.zip_hash(zf) assert zip_hash == dir_hash assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_unwriteable_contents(): my_app_setup_py = dedent(""" from setuptools import setup setup( name='my_app', version='0.0.0', zip_safe=True, packages=['my_app'], include_package_data=True, package_data={'my_app': ['unwriteable.so']}, ) """) UNWRITEABLE_PERMS = 0o400 with temporary_content( { 'setup.py': my_app_setup_py, 'my_app/__init__.py': '', 'my_app/unwriteable.so': '' }, perms=UNWRITEABLE_PERMS) as my_app_project_dir: with pushd(my_app_project_dir): subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel']) uses_my_app_setup_py = dedent(""" from setuptools import setup setup( name='uses_my_app', version='0.0.0', zip_safe=True, install_requires=['my_app'], ) """) with temporary_content({'setup.py': uses_my_app_setup_py }) as uses_my_app_project_dir: with pushd(uses_my_app_project_dir): subprocess.check_call([ sys.executable, 'setup.py', 'bdist_pex', '--pex-args=--disable-cache --no-pypi -f {}'.format( os.path.join(my_app_project_dir, 'dist')) ]) with open_zip('dist/uses_my_app-0.0.0.pex') as zf: unwriteable_sos = [ path for path in zf.namelist() if path.endswith('my_app/unwriteable.so') ] assert 1 == len(unwriteable_sos) unwriteable_so = unwriteable_sos.pop() zf.extract(unwriteable_so) assert UNWRITEABLE_PERMS == stat.S_IMODE( os.stat(unwriteable_so).st_mode)
def from_pex(cls, pex): # type: (str) -> PexInfo if zipfile.is_zipfile(pex): # Zip App with open_zip(pex) as zf: pex_info = zf.read(cls.PATH) elif os.path.isfile(pex): # Venv with open(os.path.join(os.path.dirname(pex), cls.PATH)) as fp: pex_info = fp.read() else: # Directory (Unzip mode or PEXBuilder.freeze) with open(os.path.join(pex, cls.PATH)) as fp: pex_info = fp.read() return cls.from_json(pex_info)
def test_project_name_and_version_fallback(tmpdir): # type: (Any) -> None def tmp_path(relpath): # type: (str) -> str return os.path.join(str(tmpdir), relpath) expected_metadata_project_name_and_version = ProjectNameAndVersion("foo", "1.2.3") pkg_info_src = tmp_path("PKG-INFO") with open(pkg_info_src, "w") as fp: fp.write("Name: {}\n".format(expected_metadata_project_name_and_version.project_name)) fp.write("Version: {}\n".format(expected_metadata_project_name_and_version.version)) sdist_path = tmp_path("bar-baz-4.5.6.tar.gz") with tarfile.open(sdist_path, mode="w:gz") as tf: # N.B.: Valid PKG-INFO at an invalid location. tf.add(pkg_info_src, arcname="PKG-INFO") with ENV.patch(PEX_EMIT_WARNINGS="True"), warnings.catch_warnings(record=True) as events: assert project_name_and_version(sdist_path, fallback_to_filename=False) is None assert 1 == len(events) warning = events[0] assert PEXWarning == warning.category assert "bar-baz-4.5.6/PKG-INFO" in str(warning.message) assert ProjectNameAndVersion("bar-baz", "4.5.6") == project_name_and_version( sdist_path, fallback_to_filename=True ) name_and_version = "eggs-7.8.9" pkf_info_path = "{}/PKG-INFO".format(name_and_version) def write_sdist_tgz(extension): sdist_path = tmp_path("{}.{}".format(name_and_version, extension)) with tarfile.open(sdist_path, mode="w:gz") as tf: tf.add(pkg_info_src, arcname=pkf_info_path) return sdist_path assert expected_metadata_project_name_and_version == project_name_and_version( write_sdist_tgz("tar.gz"), fallback_to_filename=False ) assert expected_metadata_project_name_and_version == project_name_and_version( write_sdist_tgz("sdist"), fallback_to_filename=False ) zip_sdist_path = tmp_path("{}.zip".format(name_and_version)) with open_zip(zip_sdist_path, mode="w") as zf: zf.write(pkg_info_src, arcname=pkf_info_path) assert expected_metadata_project_name_and_version == project_name_and_version( zip_sdist_path, fallback_to_filename=False )
def seed_cache( options, # type: Namespace pex, # type: PEX verbose=False, # type : bool ): # type: (...) -> str pex_path = pex.path() with TRACER.timed("Seeding local caches for {}".format(pex_path)): pex_info = pex.pex_info() def create_verbose_info(final_pex_path): # type: (str) -> Dict[str, str] return dict(pex_root=pex_info.pex_root, python=pex.interpreter.binary, pex=final_pex_path) if options.unzip: unzip_dir = pex_info.unzip_dir if unzip_dir is None: raise AssertionError( "Expected PEX-INFO for {} to have the components of an unzip directory" .format(pex_path)) with atomic_directory(unzip_dir, exclusive=True) as chroot: if not chroot.is_finalized: with TRACER.timed("Extracting {}".format(pex_path)): with open_zip(options.pex_name) as pex_zip: pex_zip.extractall(chroot.work_dir) if verbose: return json.dumps( create_verbose_info(final_pex_path=unzip_dir)) else: return "{} {}".format(pex.interpreter.binary, unzip_dir) elif options.venv: with TRACER.timed("Creating venv from {}".format(pex_path)): venv_pex = ensure_venv(pex) if verbose: return json.dumps( create_verbose_info(final_pex_path=venv_pex)) else: return venv_pex else: with TRACER.timed( "Extracting code and distributions for {}".format( pex_path)): pex.activate() pex_path = os.path.abspath(options.pex_name) if verbose: return json.dumps(create_verbose_info(final_pex_path=pex_path)) else: return pex_path
def test_chroot_zip(): with temporary_dir() as tmp: chroot = Chroot(os.path.join(tmp, "chroot")) chroot.write(b"data", "directory/subdirectory/file") zip_dst = os.path.join(tmp, "chroot.zip") chroot.zip(zip_dst) with open_zip(zip_dst) as zip: assert [ "directory/", "directory/subdirectory/", "directory/subdirectory/file", ] == sorted(zip.namelist()) assert b"" == zip.read("directory/") assert b"" == zip.read("directory/subdirectory/") assert b"data" == zip.read("directory/subdirectory/file")
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False, zip_safe=True, **kwargs): with make_installer(name=name, version=version, installer_impl=installer_impl, zip_safe=zip_safe, **kwargs) as installer: dist_location = installer.bdist() if zipped: yield DistributionHelper.distribution_from_path(dist_location) else: with temporary_dir() as td: extract_path = os.path.join(td, os.path.basename(dist_location)) with open_zip(dist_location) as zf: zf.extractall(extract_path) yield DistributionHelper.distribution_from_path(extract_path)
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False, zip_safe=True, **kwargs): with make_installer(name=name, version=version, installer_impl=installer_impl, zip_safe=zip_safe, **kwargs) as installer: dist_location = installer.bdist() if zipped: yield DistributionHelper.distribution_from_path(dist_location) else: with temporary_dir() as td: extract_path = os.path.join(td, os.path.basename(dist_location)) with open_zip(dist_location) as zf: zf.extractall(extract_path) yield DistributionHelper.distribution_from_path(extract_path)
def test_chroot_zip_symlink(): # type: () -> None with temporary_dir() as tmp: chroot = Chroot(os.path.join(tmp, "chroot")) chroot.write(b"data", "directory/subdirectory/file") chroot.write(b"data", "directory/subdirectory/file.foo") chroot.symlink( os.path.join(chroot.path(), "directory/subdirectory/file"), "directory/subdirectory/symlinked", ) cwd = os.getcwd() try: os.chdir(os.path.join(chroot.path(), "directory/subdirectory")) chroot.symlink( "file", "directory/subdirectory/rel-symlinked", ) finally: os.chdir(cwd) chroot.symlink(os.path.join(chroot.path(), "directory"), "symlinked") zip_dst = os.path.join(tmp, "chroot.zip") chroot.zip(zip_dst, exclude_file=lambda path: path.endswith(".foo")) with open_zip(zip_dst) as zip: assert [ "directory/", "directory/subdirectory/", "directory/subdirectory/file", "directory/subdirectory/rel-symlinked", "directory/subdirectory/symlinked", "symlinked/", "symlinked/subdirectory/", "symlinked/subdirectory/file", "symlinked/subdirectory/rel-symlinked", "symlinked/subdirectory/symlinked", ] == sorted(zip.namelist()) assert b"" == zip.read("directory/") assert b"" == zip.read("directory/subdirectory/") assert b"data" == zip.read("directory/subdirectory/file") assert b"data" == zip.read("directory/subdirectory/rel-symlinked") assert b"data" == zip.read("directory/subdirectory/symlinked") assert b"" == zip.read("symlinked/") assert b"" == zip.read("symlinked/subdirectory/") assert b"data" == zip.read("symlinked/subdirectory/file") assert b"data" == zip.read("symlinked/subdirectory/rel-symlinked") assert b"data" == zip.read("symlinked/subdirectory/symlinked")
def _load_internal_cache(self): """Possibly cache out the internal cache.""" internal_cache = os.path.join(self._pex, self._pex_info.internal_cache) with TRACER.timed("Searching dependency cache: %s" % internal_cache, V=2): if len(self._pex_info.distributions) == 0: # We have no .deps to load. return if os.path.isdir(self._pex): for distribution_name in self._pex_info.distributions: yield DistributionHelper.distribution_from_path( os.path.join(internal_cache, distribution_name)) else: with open_zip(self._pex) as zf: for dist in self._write_zipped_internal_cache(zf): yield dist
def _force_local(cls, pex_file, pex_info): if pex_info.code_hash is None: # Do not support force_local if code_hash is not set. (It should always be set.) return pex_file explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash) TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir) with atomic_directory(explode_dir) as explode_tmp: if explode_tmp: with TRACER.timed('Unzipping %s' % pex_file): with open_zip(pex_file) as pex_zip: pex_files = ( x for x in pex_zip.namelist() if not x.startswith(pex_builder.BOOTSTRAP_DIR) and not x.startswith(pex_info.internal_cache)) pex_zip.extractall(explode_tmp, pex_files) return explode_dir
def _hydrate_pex_file(self, hydrated_pex_file): # We extract source files into a temporary directory before creating the pex. td = tempfile.mkdtemp() with open_zip(self) as zf: # Populate the pex with the pinned requirements and distribution names & hashes. bootstrap_info = PexInfo.from_json(zf.read("BOOTSTRAP-PEX-INFO")) bootstrap_builder = PEXBuilder(pex_info=bootstrap_info, interpreter=PythonInterpreter.get()) # Populate the pex with the needed code. try: ipex_info = json.loads(zf.read("IPEX-INFO").decode("utf-8")) for path in ipex_info["code"]: unzipped_source = zf.extract(path, td) bootstrap_builder.add_source( unzipped_source, env_filename=_strip_app_code_prefix(path)) except Exception as e: raise ValueError( "Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}". format(e=e, info=json.dumps(ipex_info, indent=4))) # Perform a fully pinned intransitive resolve to hydrate the install cache. resolver_settings = ipex_info["resolver_settings"] sanitized_requirements = _sanitize_requirements( bootstrap_info.requirements) bootstrap_info = modify_pex_info(bootstrap_info, requirements=sanitized_requirements) bootstrap_builder.info = bootstrap_info resolved_distributions = resolver.resolve( requirements=bootstrap_info.requirements, cache=bootstrap_info.pex_root, platform="current", transitive=False, interpreter=bootstrap_builder.interpreter, **resolver_settings) # TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from # BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these # requirements or recognize that they should be pulled from the cache for some reason. for resolved_dist in resolved_distributions: bootstrap_builder.add_distribution(resolved_dist.distribution) bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False)
def test_get_pex_info(): with temporary_dir() as td: pb = write_simple_pex(td, 'print("hello world!")') pex_path = os.path.join(td, 'hello_world.pex') pb.build(pex_path) # from zip pex_info = get_pex_info(pex_path) with temporary_dir() as pex_td: with open_zip(pex_path, 'r') as zf: zf.extractall(pex_td) # from dir pex_info_2 = get_pex_info(pex_td) # same when encoded assert pex_info.dump() == pex_info_2.dump()
def test_get_pex_info(): with temporary_dir() as td: pb = write_simple_pex(td, 'print("hello world!")') pex_path = os.path.join(td, 'hello_world.pex') pb.build(pex_path) # from zip pex_info = get_pex_info(pex_path) with temporary_dir() as pex_td: with open_zip(pex_path, 'r') as zf: zf.extractall(pex_td) # from dir pex_info_2 = get_pex_info(pex_td) # same when encoded assert pex_info.dump() == pex_info_2.dump()
def explode_code( self, dest_dir, # type: str exclude=(), # type: Container[str] ): # type: (...) -> Iterable[Tuple[str, str]] with TRACER.timed("Unzipping {}".format(self._pex)): with open_zip(self._pex) as pex_zip: pex_files = ( name for name in pex_zip.namelist() if not name.startswith(pex_builder.BOOTSTRAP_DIR) and not name.startswith(self._pex_info.internal_cache) and name not in exclude) pex_zip.extractall(dest_dir, pex_files) return [( "{pex_file}:{zip_path}".format(pex_file=self._pex, zip_path=f), os.path.join(dest_dir, f), ) for f in pex_files]
def _load_internal_cache(cls, pex, pex_info): """Possibly cache out the internal cache.""" internal_cache = os.path.join(pex, pex_info.internal_cache) with TRACER.timed("Searching dependency cache: %s" % internal_cache, V=2): if len(pex_info.distributions) == 0: # We have no .deps to load. return if os.path.isdir(pex): search_path = [ os.path.join(internal_cache, dist_chroot) for dist_chroot in os.listdir(internal_cache) ] internal_env = Environment(search_path=search_path) for dist_name in internal_env: for dist in internal_env[dist_name]: yield dist else: with open_zip(pex) as zf: for dist in cls._write_zipped_internal_cache(zf, pex_info): yield dist
def zip_hash( cls, zip_file, # type: str relpath=None, # type: Optional[str] ): # type: (...) -> str """Return a reproducible hash of the contents of a zip; excluding all `.pyc` files.""" with open_zip(zip_file) as zf: names = sorted( filter_pyc_files( name for name in zf.namelist() if not name.endswith("/") and not relpath or name.startswith(relpath) ) ) def stream_factory(name): # type: (str) -> BinaryIO return cast("BinaryIO", zf.open(name, "r")) return cls._compute_hash(names, stream_factory)
def force_local(cls, pex_file, pex_info): if pex_info.code_hash is None: # Do not support force_local if code_hash is not set. (It should always be set.) return pex_file explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash) TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir) if not os.path.exists(explode_dir): explode_tmp = explode_dir + '.' + uuid.uuid4().hex with TRACER.timed('Unzipping %s' % pex_file): try: safe_mkdir(explode_tmp) with open_zip(pex_file) as pex_zip: pex_files = (x for x in pex_zip.namelist() if not x.startswith(pex_builder.BOOTSTRAP_DIR) and not x.startswith(PexInfo.INTERNAL_CACHE)) pex_zip.extractall(explode_tmp, pex_files) except: # noqa: T803 safe_rmtree(explode_tmp) raise TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir)) rename_if_empty(explode_tmp, explode_dir) return explode_dir
def force_local(cls, pex_file, pex_info): if pex_info.code_hash is None: # Do not support force_local if code_hash is not set. (It should always be set.) return pex_file explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash) TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir) if not os.path.exists(explode_dir): explode_tmp = explode_dir + '.' + uuid.uuid4().hex with TRACER.timed('Unzipping %s' % pex_file): try: safe_mkdir(explode_tmp) with open_zip(pex_file) as pex_zip: pex_files = (x for x in pex_zip.namelist() if not x.startswith(pex_builder.BOOTSTRAP_DIR) and not x.startswith(PexInfo.INTERNAL_CACHE)) pex_zip.extractall(explode_tmp, pex_files) except: # noqa: T803 safe_rmtree(explode_tmp) raise TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir)) rename_if_empty(explode_tmp, explode_dir) return explode_dir
def test_unwriteable_contents(): my_app_setup_py = dedent(""" from setuptools import setup setup( name='my_app', version='0.0.0', zip_safe=True, packages=['my_app'], include_package_data=True, package_data={'my_app': ['unwriteable.so']}, ) """) UNWRITEABLE_PERMS = 0o400 with temporary_content({'setup.py': my_app_setup_py, 'my_app/__init__.py': '', 'my_app/unwriteable.so': ''}, perms=UNWRITEABLE_PERMS) as my_app_project_dir: my_app_whl = WheelInstaller(my_app_project_dir).bdist() uses_my_app_setup_py = bdist_pex_setup_py(name='uses_my_app', version='0.0.0', zip_safe=True, install_requires=['my_app']) with temporary_content({'setup.py': uses_my_app_setup_py}) as uses_my_app_project_dir: pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl)) with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex: with open_zip(uses_my_app_pex) as zf: unwriteable_sos = [path for path in zf.namelist() if path.endswith('my_app/unwriteable.so')] assert 1 == len(unwriteable_sos) unwriteable_so = unwriteable_sos.pop() zf.extract(unwriteable_so, path=uses_my_app_project_dir) extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so) assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode)
def read_pexinfo_from_zip(entry_point): with open_zip(entry_point) as zf: return zf.read('PEX-INFO')
def get_dep_dist_names_from_pex(pex_path, match_prefix=''): """Given an on-disk pex, extract all of the unique first-level paths under `.deps`.""" with open_zip(pex_path) as pex_zip: dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith('.deps/')) return set(item for item in dep_gen if item.startswith(match_prefix))
def write_zipfile(directory, dest, reverse=False): with open_zip(dest, 'w') as zf: for filename, rel_filename in sorted(yield_files(directory), reverse=reverse): zf.write(filename, arcname=rel_filename) return dest
def write_zipfile(directory, dest, reverse=False): with open_zip(dest, 'w') as zf: for filename, rel_filename in sorted(yield_files(directory), reverse=reverse): zf.write(filename, arcname=rel_filename) return dest