def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache,
pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name,
name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def _add_dist_zip(self, path, dist_name):
# We need to distinguish between wheels and other zips. Most of the time,
# when we have a zip, it contains its contents in an importable form.
# But wheels don't have to be importable, so we need to force them
# into an importable shape. We can do that by installing it into its own
# wheel dir.
if dist_name.endswith("whl"):
from pex.third_party.wheel.install import WheelFile
tmp = safe_mkdtemp()
whltmp = os.path.join(tmp, dist_name)
os.mkdir(whltmp)
wf = WheelFile(path)
wf.install(overrides=self._get_installer_paths(whltmp), force=True)
for root, _, files in os.walk(whltmp):
pruned_dir = os.path.relpath(root, tmp)
for f in files:
fullpath = os.path.join(root, f)
target = os.path.join(self._pex_info.internal_cache, pruned_dir, f)
self._copy_or_link(fullpath, target)
return CacheHelper.dir_hash(whltmp)
with open_zip(path) as zf:
for name in zf.namelist():
if name.endswith('/'):
continue
target = os.path.join(self._pex_info.internal_cache, dist_name, name)
self._chroot.write(zf.read(name), target)
return CacheHelper.zip_hash(zf)
def get_dep_dist_names_from_pex(pex_path, match_prefix=""):
# type: (str, str) -> Set[str]
"""Given an on-disk pex, extract all of the unique first-level paths under `.deps`."""
with open_zip(pex_path) as pex_zip:
dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist()
if f.startswith(".deps/"))
return set(item for item in dep_gen if item.startswith(match_prefix))
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pb = write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with open_zip(p1.location, 'r') as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
pb = write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def _parse_sdist_package_info(sdist_path):
# type: (str) -> Optional[Message]
sdist_filename = _strip_sdist_path(sdist_path)
if sdist_filename is None:
return None
pkg_info_path = os.path.join(sdist_filename, Distribution.PKG_INFO)
if zipfile.is_zipfile(sdist_path):
with open_zip(sdist_path) as zip:
try:
return _parse_message(zip.read(pkg_info_path).decode("utf-8"))
except KeyError as e:
pex_warnings.warn(
"Source distribution {} did not have the expected metadata file {}: {}"
.format(sdist_path, pkg_info_path, e))
return None
if tarfile.is_tarfile(sdist_path):
with tarfile.open(sdist_path) as tf:
try:
pkg_info = tf.extractfile(pkg_info_path)
if pkg_info is None:
# N.B.: `extractfile` returns None for directories and special files.
return None
with closing(pkg_info) as fp:
return _parse_message(fp.read().decode("utf-8"))
except KeyError as e:
pex_warnings.warn(
"Source distribution {} did not have the expected metadata file {}: {}"
.format(sdist_path, pkg_info_path, e))
return None
return None
def test_unwriteable_contents():
my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=['my_app'],
include_package_data=True,
package_data={'my_app': ['unwriteable.so']},
)
""")
UNWRITEABLE_PERMS = 0o400
with temporary_content({'setup.py': my_app_setup_py,
'my_app/__init__.py': '',
'my_app/unwriteable.so': 'so contents'},
perms=UNWRITEABLE_PERMS) as my_app_project_dir:
my_app_whl = WheelBuilder(my_app_project_dir).bdist()
with make_project(name='uses_my_app', install_reqs=['my_app']) as uses_my_app_project_dir:
pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl))
with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex:
with open_zip(uses_my_app_pex) as zf:
unwriteable_sos = [path for path in zf.namelist()
if path.endswith('my_app/unwriteable.so')]
assert 1 == len(unwriteable_sos)
unwriteable_so = unwriteable_sos.pop()
zf.extract(unwriteable_so, path=uses_my_app_project_dir)
extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so)
with open(extract_dest) as fp:
assert 'so contents' == fp.read()
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pb = write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with open_zip(p1.location, 'r') as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
pb = write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def assert_dist_cache(zip_safe):
# type: (bool) -> None
with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(),
temporary_filename()) as (
pb,
pex_root,
pex_file,
):
pb.info.pex_root = pex_root
pb.build(pex_file)
with open_zip(pex_file) as zf:
dists = PEXEnvironment._write_zipped_internal_cache(
zf=zf, pex_info=pb.info)
assert len(dists) == 1
original_location = normalize(dists[0].location)
assert original_location.startswith(
normalize(pb.info.install_cache))
# Call a second time to validate idempotence of caching.
dists = PEXEnvironment._write_zipped_internal_cache(zf=None,
pex_info=pb.info)
assert len(dists) == 1
assert normalize(dists[0].location) == original_location
def seed_cache(
options, # type: Namespace
pex, # type: PEX
):
# type: (...) -> Iterable[str]
pex_path = pex.path()
with TRACER.timed("Seeding local caches for {}".format(pex_path)):
if options.unzip:
unzip_dir = pex.pex_info().unzip_dir
if unzip_dir is None:
raise AssertionError(
"Expected PEX-INFO for {} to have the components of an unzip directory"
.format(pex_path))
with atomic_directory(unzip_dir, exclusive=True) as chroot:
if chroot:
with TRACER.timed("Extracting {}".format(pex_path)):
with open_zip(options.pex_name) as pex_zip:
pex_zip.extractall(chroot)
return [pex.interpreter.binary, unzip_dir]
elif options.venv:
with TRACER.timed("Creating venv from {}".format(pex_path)):
venv_pex = ensure_venv(pex)
return [venv_pex]
else:
with TRACER.timed(
"Extracting code and distributions for {}".format(
pex_path)):
pex.activate()
return [os.path.abspath(options.pex_name)]
def add_from_requirements_pex(self, pex):
"""Add requirements from an existing pex.
:param pex: The path to an existing .pex file or unzipped pex directory.
"""
self._ensure_unfrozen("Adding from pex")
pex_info = PexInfo.from_pex(pex)
def add(location, dname, expected_dhash):
dhash = self._add_dist_dir(location, dname)
if dhash != expected_dhash:
raise self.InvalidDistribution(
"Distribution {} at {} had hash {}, expected {}".format(
dname, location, dhash, expected_dhash))
self._pex_info.add_distribution(dname, dhash)
if os.path.isfile(pex):
with open_zip(pex) as zf:
for dist_name, dist_hash in pex_info.distributions.items():
internal_dist_path = "/".join(
[pex_info.internal_cache, dist_name])
cached_location = os.path.join(pex_info.install_cache,
dist_hash, dist_name)
CacheHelper.cache_distribution(zf, internal_dist_path,
cached_location)
add(cached_location, dist_name, dist_hash)
else:
for dist_name, dist_hash in pex_info.distributions.items():
add(os.path.join(pex, pex_info.internal_cache, dist_name),
dist_name, dist_hash)
for req in pex_info.requirements:
self._pex_info.add_requirement(req)
def from_pex(cls, pex):
if os.path.isfile(pex):
with open_zip(pex) as zf:
pex_info = zf.read(cls.PATH)
else:
with open(os.path.join(pex, cls.PATH)) as fp:
pex_info = fp.read()
return cls.from_json(pex_info)
def from_pex(cls, pex):
if os.path.isfile(pex):
with open_zip(pex) as zf:
pex_info = zf.read(cls.PATH)
else:
with open(os.path.join(pex, cls.PATH)) as fp:
pex_info = fp.read()
return cls.from_json(pex_info)
def _parse_wheel_package_info(wheel_path):
# type: (str) -> Optional[Message]
if not wheel_path.endswith(".whl") or not zipfile.is_zipfile(wheel_path):
return None
project_name, version, _ = os.path.basename(wheel_path).split("-", 2)
dist_info_dir = "{}-{}.dist-info".format(project_name, version)
with open_zip(wheel_path) as whl:
with whl.open(os.path.join(dist_info_dir, DistInfoDistribution.PKG_INFO)) as fp:
return _parse_message(fp.read())
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with open_zip(tf.name, 'r') as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with open_zip(tf.name, 'r') as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_unwriteable_contents():
my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=['my_app'],
include_package_data=True,
package_data={'my_app': ['unwriteable.so']},
)
""")
UNWRITEABLE_PERMS = 0o400
with temporary_content(
{
'setup.py': my_app_setup_py,
'my_app/__init__.py': '',
'my_app/unwriteable.so': ''
},
perms=UNWRITEABLE_PERMS) as my_app_project_dir:
with pushd(my_app_project_dir):
subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel'])
uses_my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='uses_my_app',
version='0.0.0',
zip_safe=True,
install_requires=['my_app'],
)
""")
with temporary_content({'setup.py': uses_my_app_setup_py
}) as uses_my_app_project_dir:
with pushd(uses_my_app_project_dir):
subprocess.check_call([
sys.executable, 'setup.py', 'bdist_pex',
'--pex-args=--disable-cache --no-pypi -f {}'.format(
os.path.join(my_app_project_dir, 'dist'))
])
with open_zip('dist/uses_my_app-0.0.0.pex') as zf:
unwriteable_sos = [
path for path in zf.namelist()
if path.endswith('my_app/unwriteable.so')
]
assert 1 == len(unwriteable_sos)
unwriteable_so = unwriteable_sos.pop()
zf.extract(unwriteable_so)
assert UNWRITEABLE_PERMS == stat.S_IMODE(
os.stat(unwriteable_so).st_mode)
def from_pex(cls, pex):
# type: (str) -> PexInfo
if zipfile.is_zipfile(pex): # Zip App
with open_zip(pex) as zf:
pex_info = zf.read(cls.PATH)
elif os.path.isfile(pex): # Venv
with open(os.path.join(os.path.dirname(pex), cls.PATH)) as fp:
pex_info = fp.read()
else: # Directory (Unzip mode or PEXBuilder.freeze)
with open(os.path.join(pex, cls.PATH)) as fp:
pex_info = fp.read()
return cls.from_json(pex_info)
def test_project_name_and_version_fallback(tmpdir):
# type: (Any) -> None
def tmp_path(relpath):
# type: (str) -> str
return os.path.join(str(tmpdir), relpath)
expected_metadata_project_name_and_version = ProjectNameAndVersion("foo", "1.2.3")
pkg_info_src = tmp_path("PKG-INFO")
with open(pkg_info_src, "w") as fp:
fp.write("Name: {}\n".format(expected_metadata_project_name_and_version.project_name))
fp.write("Version: {}\n".format(expected_metadata_project_name_and_version.version))
sdist_path = tmp_path("bar-baz-4.5.6.tar.gz")
with tarfile.open(sdist_path, mode="w:gz") as tf:
# N.B.: Valid PKG-INFO at an invalid location.
tf.add(pkg_info_src, arcname="PKG-INFO")
with ENV.patch(PEX_EMIT_WARNINGS="True"), warnings.catch_warnings(record=True) as events:
assert project_name_and_version(sdist_path, fallback_to_filename=False) is None
assert 1 == len(events)
warning = events[0]
assert PEXWarning == warning.category
assert "bar-baz-4.5.6/PKG-INFO" in str(warning.message)
assert ProjectNameAndVersion("bar-baz", "4.5.6") == project_name_and_version(
sdist_path, fallback_to_filename=True
)
name_and_version = "eggs-7.8.9"
pkf_info_path = "{}/PKG-INFO".format(name_and_version)
def write_sdist_tgz(extension):
sdist_path = tmp_path("{}.{}".format(name_and_version, extension))
with tarfile.open(sdist_path, mode="w:gz") as tf:
tf.add(pkg_info_src, arcname=pkf_info_path)
return sdist_path
assert expected_metadata_project_name_and_version == project_name_and_version(
write_sdist_tgz("tar.gz"), fallback_to_filename=False
)
assert expected_metadata_project_name_and_version == project_name_and_version(
write_sdist_tgz("sdist"), fallback_to_filename=False
)
zip_sdist_path = tmp_path("{}.zip".format(name_and_version))
with open_zip(zip_sdist_path, mode="w") as zf:
zf.write(pkg_info_src, arcname=pkf_info_path)
assert expected_metadata_project_name_and_version == project_name_and_version(
zip_sdist_path, fallback_to_filename=False
)
def seed_cache(
options, # type: Namespace
pex, # type: PEX
verbose=False, # type : bool
):
# type: (...) -> str
pex_path = pex.path()
with TRACER.timed("Seeding local caches for {}".format(pex_path)):
pex_info = pex.pex_info()
def create_verbose_info(final_pex_path):
# type: (str) -> Dict[str, str]
return dict(pex_root=pex_info.pex_root,
python=pex.interpreter.binary,
pex=final_pex_path)
if options.unzip:
unzip_dir = pex_info.unzip_dir
if unzip_dir is None:
raise AssertionError(
"Expected PEX-INFO for {} to have the components of an unzip directory"
.format(pex_path))
with atomic_directory(unzip_dir, exclusive=True) as chroot:
if not chroot.is_finalized:
with TRACER.timed("Extracting {}".format(pex_path)):
with open_zip(options.pex_name) as pex_zip:
pex_zip.extractall(chroot.work_dir)
if verbose:
return json.dumps(
create_verbose_info(final_pex_path=unzip_dir))
else:
return "{} {}".format(pex.interpreter.binary, unzip_dir)
elif options.venv:
with TRACER.timed("Creating venv from {}".format(pex_path)):
venv_pex = ensure_venv(pex)
if verbose:
return json.dumps(
create_verbose_info(final_pex_path=venv_pex))
else:
return venv_pex
else:
with TRACER.timed(
"Extracting code and distributions for {}".format(
pex_path)):
pex.activate()
pex_path = os.path.abspath(options.pex_name)
if verbose:
return json.dumps(create_verbose_info(final_pex_path=pex_path))
else:
return pex_path
def test_chroot_zip():
with temporary_dir() as tmp:
chroot = Chroot(os.path.join(tmp, "chroot"))
chroot.write(b"data", "directory/subdirectory/file")
zip_dst = os.path.join(tmp, "chroot.zip")
chroot.zip(zip_dst)
with open_zip(zip_dst) as zip:
assert [
"directory/",
"directory/subdirectory/",
"directory/subdirectory/file",
] == sorted(zip.namelist())
assert b"" == zip.read("directory/")
assert b"" == zip.read("directory/subdirectory/")
assert b"data" == zip.read("directory/subdirectory/file")
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False,
zip_safe=True, **kwargs):
with make_installer(name=name,
version=version,
installer_impl=installer_impl,
zip_safe=zip_safe,
**kwargs) as installer:
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with open_zip(dist_location) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False,
zip_safe=True, **kwargs):
with make_installer(name=name,
version=version,
installer_impl=installer_impl,
zip_safe=zip_safe,
**kwargs) as installer:
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with open_zip(dist_location) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def test_chroot_zip_symlink():
# type: () -> None
with temporary_dir() as tmp:
chroot = Chroot(os.path.join(tmp, "chroot"))
chroot.write(b"data", "directory/subdirectory/file")
chroot.write(b"data", "directory/subdirectory/file.foo")
chroot.symlink(
os.path.join(chroot.path(), "directory/subdirectory/file"),
"directory/subdirectory/symlinked",
)
cwd = os.getcwd()
try:
os.chdir(os.path.join(chroot.path(), "directory/subdirectory"))
chroot.symlink(
"file",
"directory/subdirectory/rel-symlinked",
)
finally:
os.chdir(cwd)
chroot.symlink(os.path.join(chroot.path(), "directory"), "symlinked")
zip_dst = os.path.join(tmp, "chroot.zip")
chroot.zip(zip_dst, exclude_file=lambda path: path.endswith(".foo"))
with open_zip(zip_dst) as zip:
assert [
"directory/",
"directory/subdirectory/",
"directory/subdirectory/file",
"directory/subdirectory/rel-symlinked",
"directory/subdirectory/symlinked",
"symlinked/",
"symlinked/subdirectory/",
"symlinked/subdirectory/file",
"symlinked/subdirectory/rel-symlinked",
"symlinked/subdirectory/symlinked",
] == sorted(zip.namelist())
assert b"" == zip.read("directory/")
assert b"" == zip.read("directory/subdirectory/")
assert b"data" == zip.read("directory/subdirectory/file")
assert b"data" == zip.read("directory/subdirectory/rel-symlinked")
assert b"data" == zip.read("directory/subdirectory/symlinked")
assert b"" == zip.read("symlinked/")
assert b"" == zip.read("symlinked/subdirectory/")
assert b"data" == zip.read("symlinked/subdirectory/file")
assert b"data" == zip.read("symlinked/subdirectory/rel-symlinked")
assert b"data" == zip.read("symlinked/subdirectory/symlinked")
def _load_internal_cache(self):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(self._pex, self._pex_info.internal_cache)
with TRACER.timed("Searching dependency cache: %s" % internal_cache,
V=2):
if len(self._pex_info.distributions) == 0:
# We have no .deps to load.
return
if os.path.isdir(self._pex):
for distribution_name in self._pex_info.distributions:
yield DistributionHelper.distribution_from_path(
os.path.join(internal_cache, distribution_name))
else:
with open_zip(self._pex) as zf:
for dist in self._write_zipped_internal_cache(zf):
yield dist
def _force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache,
pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
with atomic_directory(explode_dir) as explode_tmp:
if explode_tmp:
with TRACER.timed('Unzipping %s' % pex_file):
with open_zip(pex_file) as pex_zip:
pex_files = (
x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR)
and not x.startswith(pex_info.internal_cache))
pex_zip.extractall(explode_tmp, pex_files)
return explode_dir
def _hydrate_pex_file(self, hydrated_pex_file):
# We extract source files into a temporary directory before creating the pex.
td = tempfile.mkdtemp()
with open_zip(self) as zf:
# Populate the pex with the pinned requirements and distribution names & hashes.
bootstrap_info = PexInfo.from_json(zf.read("BOOTSTRAP-PEX-INFO"))
bootstrap_builder = PEXBuilder(pex_info=bootstrap_info,
interpreter=PythonInterpreter.get())
# Populate the pex with the needed code.
try:
ipex_info = json.loads(zf.read("IPEX-INFO").decode("utf-8"))
for path in ipex_info["code"]:
unzipped_source = zf.extract(path, td)
bootstrap_builder.add_source(
unzipped_source, env_filename=_strip_app_code_prefix(path))
except Exception as e:
raise ValueError(
"Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}".
format(e=e, info=json.dumps(ipex_info, indent=4)))
# Perform a fully pinned intransitive resolve to hydrate the install cache.
resolver_settings = ipex_info["resolver_settings"]
sanitized_requirements = _sanitize_requirements(
bootstrap_info.requirements)
bootstrap_info = modify_pex_info(bootstrap_info,
requirements=sanitized_requirements)
bootstrap_builder.info = bootstrap_info
resolved_distributions = resolver.resolve(
requirements=bootstrap_info.requirements,
cache=bootstrap_info.pex_root,
platform="current",
transitive=False,
interpreter=bootstrap_builder.interpreter,
**resolver_settings)
# TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from
# BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these
# requirements or recognize that they should be pulled from the cache for some reason.
for resolved_dist in resolved_distributions:
bootstrap_builder.add_distribution(resolved_dist.distribution)
bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False)
def test_get_pex_info():
with temporary_dir() as td:
pb = write_simple_pex(td, 'print("hello world!")')
pex_path = os.path.join(td, 'hello_world.pex')
pb.build(pex_path)
# from zip
pex_info = get_pex_info(pex_path)
with temporary_dir() as pex_td:
with open_zip(pex_path, 'r') as zf:
zf.extractall(pex_td)
# from dir
pex_info_2 = get_pex_info(pex_td)
# same when encoded
assert pex_info.dump() == pex_info_2.dump()
def test_get_pex_info():
with temporary_dir() as td:
pb = write_simple_pex(td, 'print("hello world!")')
pex_path = os.path.join(td, 'hello_world.pex')
pb.build(pex_path)
# from zip
pex_info = get_pex_info(pex_path)
with temporary_dir() as pex_td:
with open_zip(pex_path, 'r') as zf:
zf.extractall(pex_td)
# from dir
pex_info_2 = get_pex_info(pex_td)
# same when encoded
assert pex_info.dump() == pex_info_2.dump()
def explode_code(
self,
dest_dir, # type: str
exclude=(), # type: Container[str]
):
# type: (...) -> Iterable[Tuple[str, str]]
with TRACER.timed("Unzipping {}".format(self._pex)):
with open_zip(self._pex) as pex_zip:
pex_files = (
name for name in pex_zip.namelist()
if not name.startswith(pex_builder.BOOTSTRAP_DIR)
and not name.startswith(self._pex_info.internal_cache)
and name not in exclude)
pex_zip.extractall(dest_dir, pex_files)
return [(
"{pex_file}:{zip_path}".format(pex_file=self._pex,
zip_path=f),
os.path.join(dest_dir, f),
) for f in pex_files]
def _load_internal_cache(cls, pex, pex_info):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(pex, pex_info.internal_cache)
with TRACER.timed("Searching dependency cache: %s" % internal_cache, V=2):
if len(pex_info.distributions) == 0:
# We have no .deps to load.
return
if os.path.isdir(pex):
search_path = [
os.path.join(internal_cache, dist_chroot)
for dist_chroot in os.listdir(internal_cache)
]
internal_env = Environment(search_path=search_path)
for dist_name in internal_env:
for dist in internal_env[dist_name]:
yield dist
else:
with open_zip(pex) as zf:
for dist in cls._write_zipped_internal_cache(zf, pex_info):
yield dist
def zip_hash(
cls,
zip_file, # type: str
relpath=None, # type: Optional[str]
):
# type: (...) -> str
"""Return a reproducible hash of the contents of a zip; excluding all `.pyc` files."""
with open_zip(zip_file) as zf:
names = sorted(
filter_pyc_files(
name
for name in zf.namelist()
if not name.endswith("/") and not relpath or name.startswith(relpath)
)
)
def stream_factory(name):
# type: (str) -> BinaryIO
return cast("BinaryIO", zf.open(name, "r"))
return cls._compute_hash(names, stream_factory)
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def test_unwriteable_contents():
my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=['my_app'],
include_package_data=True,
package_data={'my_app': ['unwriteable.so']},
)
""")
UNWRITEABLE_PERMS = 0o400
with temporary_content({'setup.py': my_app_setup_py,
'my_app/__init__.py': '',
'my_app/unwriteable.so': ''},
perms=UNWRITEABLE_PERMS) as my_app_project_dir:
my_app_whl = WheelInstaller(my_app_project_dir).bdist()
uses_my_app_setup_py = bdist_pex_setup_py(name='uses_my_app',
version='0.0.0',
zip_safe=True,
install_requires=['my_app'])
with temporary_content({'setup.py': uses_my_app_setup_py}) as uses_my_app_project_dir:
pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl))
with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex:
with open_zip(uses_my_app_pex) as zf:
unwriteable_sos = [path for path in zf.namelist()
if path.endswith('my_app/unwriteable.so')]
assert 1 == len(unwriteable_sos)
unwriteable_so = unwriteable_sos.pop()
zf.extract(unwriteable_so, path=uses_my_app_project_dir)
extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so)
assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode)
def read_pexinfo_from_zip(entry_point):
with open_zip(entry_point) as zf:
return zf.read('PEX-INFO')
def get_dep_dist_names_from_pex(pex_path, match_prefix=''):
"""Given an on-disk pex, extract all of the unique first-level paths under `.deps`."""
with open_zip(pex_path) as pex_zip:
dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith('.deps/'))
return set(item for item in dep_gen if item.startswith(match_prefix))
def write_zipfile(directory, dest, reverse=False):
with open_zip(dest, 'w') as zf:
for filename, rel_filename in sorted(yield_files(directory), reverse=reverse):
zf.write(filename, arcname=rel_filename)
return dest
def write_zipfile(directory, dest, reverse=False):
with open_zip(dest, 'w') as zf:
for filename, rel_filename in sorted(yield_files(directory), reverse=reverse):
zf.write(filename, arcname=rel_filename)
return dest
