def test_pex_builder_from_requirements_pex():
def build_from_req_pex(path, req_pex):
pb = PEXBuilder(path=path)
pb.add_from_requirements_pex(req_pex)
with open(os.path.join(path, 'exe.py'), 'w') as fp:
fp.write(exe_main)
pb.set_executable(os.path.join(path, 'exe.py'))
pb.freeze()
return pb
def verify(pb):
success_txt = os.path.join(pb.path(), 'success.txt')
PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# Build from pex dir.
with temporary_dir() as td2:
with nested(temporary_dir(), make_bdist('p1')) as (td1, p1):
pb1 = write_pex(td1, dists=[p1])
pb2 = build_from_req_pex(td2, pb1.path())
verify(pb2)
# Build from .pex file.
with temporary_dir() as td4:
with nested(temporary_dir(), make_bdist('p1')) as (td3, p1):
pb3 = write_pex(td3, dists=[p1])
target = os.path.join(td3, 'foo.pex')
pb3.build(target)
pb4 = build_from_req_pex(td4, target)
verify(pb4)
def test_pex_builder_from_requirements_pex():
# type: () -> None
def build_from_req_pex(path, req_pex):
# type: (str, PEXBuilder) -> PEXBuilder
pb = PEXBuilder(path=path)
pb.add_from_requirements_pex(req_pex)
with open(os.path.join(path, "exe.py"), "w") as fp:
fp.write(exe_main)
pb.set_executable(os.path.join(path, "exe.py"))
pb.freeze()
return pb
def verify(pb):
# type: (PEXBuilder) -> None
success_txt = os.path.join(pb.path(), "success.txt")
PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == "success"
# Build from pex dir.
with temporary_dir() as td2:
with nested(temporary_dir(), make_bdist("p1")) as (td1, p1):
pb1 = write_pex(td1, dists=[p1])
pb2 = build_from_req_pex(td2, pb1.path())
verify(pb2)
# Build from .pex file.
with temporary_dir() as td4:
with nested(temporary_dir(), make_bdist("p1")) as (td3, p1):
pb3 = write_pex(td3, dists=[p1])
target = os.path.join(td3, "foo.pex")
pb3.build(target)
pb4 = build_from_req_pex(td4, target)
verify(pb4)
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(),
temporary_file()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file.name)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file.name, pb.info)
assert len(zip_safe) == 1
assert normalize(zip_safe[0].location).startswith(
normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
'loc: %s, cache: %s' %
(normalize(zip_safe[0].location),
normalize(os.path.join(pex_file.name,
pb.info.internal_cache))))
pb.info.always_write_cache = True
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file.name, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(
normalize(pb.info.install_cache))
# Check that we can read from the cache
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file.name, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location).startswith(
normalize(pb.info.install_cache))
# non-zip_safe pex will be written to install cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(),
temporary_file()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file.name)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file.name, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(
normalize(pb.info.install_cache))
original_location = normalize(new[0].location)
# do the second time to validate idempotence of caching
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file.name, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location) == original_location
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def assert_dist_cache(zip_safe):
# type: (bool) -> None
with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(),
temporary_filename()) as (
pb,
pex_root,
pex_file,
):
pb.info.pex_root = pex_root
pb.build(pex_file)
with open_zip(pex_file) as zf:
dists = PEXEnvironment._write_zipped_internal_cache(
zf=zf, pex_info=pb.info)
assert len(dists) == 1
original_location = normalize(dists[0].location)
assert original_location.startswith(
normalize(pb.info.install_cache))
# Call a second time to validate idempotence of caching.
dists = PEXEnvironment._write_zipped_internal_cache(zf=None,
pex_info=pb.info)
assert len(dists) == 1
assert normalize(dists[0].location) == original_location
def test_pex_builder_compilation():
# type: () -> None
with nested(temporary_dir(), temporary_dir(),
temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, "src.py")
with open(src, "w") as fp:
fp.write(exe_main)
exe = os.path.join(td1, "exe.py")
with open(exe, "w") as fp:
fp.write(exe_main)
def build_and_check(path, precompile):
# type: (str, bool) -> None
pb = PEXBuilder(path=path)
pb.add_source(src, "lib/src.py")
pb.set_executable(exe, "exe.py")
pb.freeze(bytecode_compile=precompile)
for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, BOOTSTRAP_DIR)
bootstrap_pycs = [] # type: List[str]
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith(".pyc"))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
build_and_check(td2, False)
build_and_check(td3, True)
def yield_pex_builder(zip_safe=True, interpreter=None):
with nested(temporary_dir(),
make_bdist('p1', zip_safe=zip_safe,
interpreter=interpreter)) as (td, p1):
pb = PEXBuilder(path=td, interpreter=interpreter)
pb.add_dist_location(p1.location)
yield pb
def test_force_local():
# type: () -> None
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
pb,
pex_root,
pex_file,
):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment._force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
listing = set(os.listdir(pb.info.zip_unsafe_cache))
# The code_cache should be a write-locked directory.
assert len(listing) == 2
listing.remove(os.path.basename(code_cache))
lockfile = listing.pop()
assert os.path.isfile(os.path.join(pb.info.zip_unsafe_cache, lockfile))
assert set(os.listdir(code_cache)) == {PexInfo.PATH, "__main__.py", "__main__.pyc"}
# idempotence
assert PEXEnvironment._force_local(pex_file, pb.info) == code_cache
def test_access_zipped_assets():
try:
import __builtin__
builtin_path = __builtin__.__name__
except ImportError:
# Python3
import builtins
builtin_path = builtins.__name__
mock_open = mock.mock_open()
with nested(mock.patch('%s.open' % builtin_path, mock_open, create=True),
mock.patch('pex.util.resource_string', autospec=True, spec_set=True),
mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True),
mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True),
mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True)) as (
_, mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdtemp):
mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir'])
mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']])
mock_resource_isdir.side_effect = iter([False, True, False])
mock_resource_string.return_value = 'testing'
temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil')
assert mock_resource_listdir.call_count == 2
assert mock_open.call_count == 2
file_handle = mock_open.return_value.__enter__.return_value
assert file_handle.write.call_count == 2
assert mock_safe_mkdtemp.mock_calls == [mock.call()]
assert temp_dir == 'tmpJIMMEH'
def test_resolve_cache():
# type: () -> None
project_wheel = build_wheel(name="project")
with nested(temporary_dir(), temporary_dir()) as (td, cache):
safe_copy(project_wheel,
os.path.join(td, os.path.basename(project_wheel)))
# Without a cache, each resolve should be isolated, but otherwise identical.
resolved_dists1 = local_resolve_multi(["project"], find_links=[td])
resolved_dists2 = local_resolve_multi(["project"], find_links=[td])
assert resolved_dists1 != resolved_dists2
assert len(resolved_dists1) == 1
assert len(resolved_dists2) == 1
assert resolved_dists1[0].requirement == resolved_dists2[0].requirement
assert resolved_dists1[0].distribution.location != resolved_dists2[
0].distribution.location
# With a cache, each resolve should be identical.
resolved_dists3 = local_resolve_multi(["project"],
find_links=[td],
cache=cache)
resolved_dists4 = local_resolve_multi(["project"],
find_links=[td],
cache=cache)
assert resolved_dists1 != resolved_dists3
assert resolved_dists2 != resolved_dists3
assert resolved_dists3 == resolved_dists4
def test_pex_builder_copy_or_link():
# type: () -> None
with nested(temporary_dir(), temporary_dir(),
temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, "exe.py")
with open(src, "w") as fp:
fp.write(exe_main)
def build_and_check(path, copy):
# type: (str, bool) -> None
pb = PEXBuilder(path=path, copy=copy)
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO],
s1[stat.ST_DEV]) == (s2[stat.ST_INO],
s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
build_and_check(td2, False)
build_and_check(td3, True)
def test_pex_builder_compilation():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, 'src.py')
with open(src, 'w') as fp:
fp.write(exe_main)
exe = os.path.join(td1, 'exe.py')
with open(exe, 'w') as fp:
fp.write(exe_main)
def build_and_check(path, precompile):
pb = PEXBuilder(path)
pb.add_source(src, 'lib/src.py')
pb.set_executable(exe, 'exe.py')
pb.freeze(bytecode_compile=precompile)
for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
bootstrap_pycs = []
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
build_and_check(td2, False)
build_and_check(td3, True)
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def test_osx_platform_intel_issue_523():
def bad_interpreter():
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
# setuptools and wheel extras.
with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=pb.interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
def run(args, **env):
pex_env = os.environ.copy()
pex_env['PEX_VERBOSE'] = '1'
pex_env.update(**env)
process = pex.run(args=args,
env=pex_env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
return process.returncode, stdout, stderr
returncode, _, stderr = run(['-c', 'import psutil'])
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
returncode, stdout, stderr = run(['-c', 'import pkg_resources'])
assert 0 != returncode, (
'Isolated pex process succeeded but should not have found pkg-resources:\n'
'STDOUT:\n'
'{}\n'
'STDERR:\n'
'{}'
.format(stdout, stdout, stderr)
)
returncode, stdout, stderr = run(
['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'],
# Let the bad interpreter site-packages setuptools leak in.
PEX_INHERIT_PATH='1'
)
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
# Verify this worked along side the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex, interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def test_site_libs():
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages])
site_libs = PEX.site_libs()
assert site_packages in site_libs
def test_site_libs():
with nested(mock.patch.object(PEX, '_get_site_packages'),
temporary_dir()) as (mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages])
site_libs = PEX.site_libs()
assert site_packages in site_libs
def test_clp_prereleases_resolver():
# type: () -> None
with nested(
built_wheel(name="prerelease-dep", version="1.2.3b1"),
built_wheel(name="transitive-dep",
install_reqs=["prerelease-dep"]),
built_wheel(name="dep",
install_reqs=["prerelease-dep>=1.2",
"transitive-dep"]),
temporary_dir(),
temporary_dir(),
) as (prerelease_dep, transitive_dep, dep, dist_dir, cache_dir):
for dist in (prerelease_dep, transitive_dep, dep):
safe_copy(dist, os.path.join(dist_dir, os.path.basename(dist)))
parser = configure_clp()
options, reqs = parser.parse_args(args=[
"--no-index",
"--find-links",
dist_dir,
"--cache-dir",
cache_dir, # Avoid dangling {pex_root}.
"--no-pre",
"dep",
])
assert not options.allow_prereleases
with pytest.raises(
SystemExit,
message="Should have failed to resolve prerelease dep"):
build_pex(reqs, options)
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(args=[
"--no-index",
"--find-links",
dist_dir,
"--cache-dir",
cache_dir, # Avoid dangling {pex_root}.
"--pre",
"dep",
])
assert options.allow_prereleases
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
pex_builder = build_pex(reqs, options)
assert pex_builder is not None
assert len(
pex_builder.info.distributions) == 3, "Should have resolved deps"
def test_pex_root():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td, output_dir, tmp_home):
output_path = os.path.join(output_dir, 'pex.pex')
args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)]
results = run_pex_command(args=args, env=make_env(HOME=tmp_home, PEX_INTERPRETER='1'))
results.assert_success()
assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.'
assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.'
assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
def yield_pex_builder(zip_safe=True, interpreter=None):
# type: (bool, Optional[PythonInterpreter]) -> Iterator[PEXBuilder]
with nested(temporary_dir(), make_bdist("p1", zip_safe=zip_safe, interpreter=interpreter)) as (
td,
p1,
):
pb = PEXBuilder(path=td, interpreter=interpreter)
pb.add_dist_location(p1.location)
yield pb
def test_load_internal_cache_unzipped():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir()) as (pb, pex_root):
pb.info.pex_root = pex_root
pb.freeze()
dists = list(PEXEnvironment.load_internal_cache(pb.path(), pb.info))
assert len(dists) == 1
assert normalize(dists[0].location).startswith(normalize(os.path.join(pb.path(), pb.info.internal_cache)))
def test_osx_platform_intel_issue_523():
def bad_interpreter(include_site_extras=True):
return PythonInterpreter.from_binary(
_KNOWN_BAD_APPLE_INTERPRETER,
include_site_extras=include_site_extras)
interpreter = bad_interpreter(include_site_extras=False)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for resolved_dist in resolver.resolve(
[requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version,
dist.location)
with nested(
yield_pex_builder(installer_impl=WheelInstaller,
interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage,
WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
args = [
'-c',
'import pkg_resources; print(pkg_resources.get_supported_platform())'
]
env = os.environ.copy()
env['PEX_VERBOSE'] = '1'
process = pex.run(args=args,
env=env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(
process.returncode, stderr))
# Verify this all worked under the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes(
'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_load_internal_cache_unzipped():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir()) as (pb, pex_root):
pb.info.pex_root = pex_root
pb.freeze()
dists = list(PEXEnvironment.load_internal_cache(pb.path(), pb.info))
assert len(dists) == 1
assert normalize(dists[0].location).startswith(
normalize(os.path.join(pb.path(), pb.info.internal_cache)))
def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file.name)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(zip_safe) == 1
assert normalize(zip_safe[0].location).startswith(
normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
'loc: %s, cache: %s' % (
normalize(zip_safe[0].location),
normalize(os.path.join(pex_file.name, pb.info.internal_cache))))
pb.info.always_write_cache = True
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
# Check that we can read from the cache
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location).startswith(normalize(pb.info.install_cache))
# non-zip_safe pex will be written to install cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file.name)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
original_location = normalize(new[0].location)
# do the second time to validate idempotence of caching
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location) == original_location
def test_site_libs_excludes_prefix():
"""Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages, sys.prefix])
site_libs = PEX.site_libs()
assert site_packages in site_libs
assert sys.prefix not in site_libs
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1')) as (td, p1):
pb = write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1')) as (td1, td2, p1):
pb = write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_site_libs():
# type: () -> None
with nested(mock.patch.object(PEX, "_get_site_packages"), temporary_dir()) as (
mock_site_packages,
tempdir,
):
site_packages = os.path.join(tempdir, "site-packages")
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages])
site_libs = PEX.site_libs()
assert site_packages in site_libs
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of something from a wheel that
doesn't come in importable form."""
with nested(temporary_dir(), make_bdist("p1")) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, pyparsing_path])
success_txt = os.path.join(td, "success.txt")
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == "success"
def test_site_libs_excludes_prefix():
"""Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""
with nested(mock.patch.object(PEX, '_get_site_packages'),
temporary_dir()) as (mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages, sys.prefix])
site_libs = PEX.site_libs()
assert site_packages in site_libs
assert sys.prefix not in site_libs
def test_load_internal_cache_unzipped():
# Unzipped pexes should use distributions from the pex internal cache.
with nested(yield_pex_builder(zip_safe=True),
temporary_dir()) as (pb, pex_root):
pb.info.pex_root = pex_root
pb.freeze()
dists = list(PEXEnvironment._load_internal_cache(pb.path(), pb.info))
assert len(dists) == 1
assert normalize(dists[0].location).startswith(
normalize(os.path.join(pb.path(), pb.info.internal_cache)))
def test_site_libs_symlink():
with nested(mock.patch.object(PEX, '_get_site_packages'),
temporary_dir()) as (mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
site_packages_link = os.path.join(tempdir, 'site-packages-link')
os.symlink(site_packages, site_packages_link)
mock_site_packages.return_value = set([site_packages_link])
site_libs = PEX.site_libs()
assert os.path.realpath(site_packages) in site_libs
assert site_packages_link in site_libs
def test_site_libs_symlink():
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
site_packages_link = os.path.join(tempdir, 'site-packages-link')
os.symlink(site_packages, site_packages_link)
mock_site_packages.return_value = set([site_packages_link])
site_libs = PEX.site_libs()
assert os.path.realpath(site_packages) in site_libs
assert site_packages_link in site_libs
def test_force_local():
with nested(yield_pex_builder(), temporary_dir(), temporary_file()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file.name)
code_cache = PEXEnvironment.force_local(pex_file.name, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py'])
# idempotence
assert PEXEnvironment.force_local(pex_file.name, pb.info) == code_cache
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_force_local():
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment.force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
assert set(os.listdir(code_cache)) == set([PexInfo.PATH, "__main__.py", "__main__.pyc"])
# idempotence
assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pex = pb.path()
pb.info.pex_root = pex_root
pb.build(pex_file.name)
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(
normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
'loc: %s, cache: %s' % (
normalize(dists[0].location),
normalize(os.path.join(pex_file.name, pb.info.internal_cache))))
pb.info.always_write_cache = True
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pex = pb.path()
pb.info.pex_root = pex_root
pb.build(pex_file.name)
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
original_location = normalize(dists[0].location)
# do the second time to validate idempotence of caching
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location) == original_location
def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pex = pb.path()
pb.info.pex_root = pex_root
pb.build(pex_file.name)
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(
normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
'loc: %s, cache: %s' % (
normalize(dists[0].location),
normalize(os.path.join(pex_file.name, pb.info.internal_cache))))
pb.info.always_write_cache = True
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
pb, pex_root, pex_file):
pex = pb.path()
pb.info.pex_root = pex_root
pb.build(pex_file.name)
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
original_location = normalize(dists[0].location)
# do the second time to validate idempotence of caching
dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
assert len(dists) == 1
assert normalize(dists[0].location) == original_location
def run_simple_pex_test(body,
args=(),
env=None,
dists=None,
coverage=False,
interpreter=None):
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = write_simple_pex(td1,
body,
dists=dists,
coverage=coverage,
interpreter=interpreter)
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
return run_simple_pex(pex, args=args, env=env, interpreter=interpreter)
def test_pex_builder_copy_or_link():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, 'exe.py')
with open(src, 'w') as fp:
fp.write(exe_main)
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
s1 = os.stat(src)
s2 = os.stat(os.path.join(path, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
build_and_check(td2, False)
build_and_check(td3, True)
def yield_pex_builder(zip_safe=True):
with nested(temporary_dir(), make_bdist("p1", zipped=True, zip_safe=zip_safe)) as (td, p1):
pb = PEXBuilder(path=td)
pb.add_egg(p1.location)
yield pb
def test_clp_prereleases_resolver():
prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
with nested(temporary_dir(), temporary_dir()) as (dist_dir, cache_dir):
safe_copy(prerelease_dep, os.path.join(dist_dir, os.path.basename(prerelease_dep)))
fetcher = Fetcher([dist_dir])
# When no specific options are specified, allow_prereleases is None
parser, resolver_options_builder = configure_clp()
assert resolver_options_builder._allow_prereleases is None
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(args=['--cache-dir', cache_dir, # Avoid dangling {pex_root}.
'--pre', 'dep==1.2.3b1', 'dep'])
assert resolver_options_builder._allow_prereleases
# We need to use our own fetcher instead of PyPI
resolver_options_builder._fetchers.insert(0, fetcher)
#####
# The resolver created during processing of command line options (configure_clp)
# is not actually passed into the API call (resolve_multi) from build_pex().
# Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
# is created. The only way to supply our own fetcher to that new instance is to patch it
# here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
# below and the change in the `pex.resolver` module where the patched object resides.
#
import pex.resolver
class BuilderWithFetcher(ResolverOptionsBuilder):
def __init__(self,
fetchers=None,
allow_all_external=False,
allow_external=None,
allow_unverified=None,
allow_prereleases=None,
use_manylinux=None,
precedence=None,
context=None
):
super(BuilderWithFetcher, self).__init__(fetchers=fetchers,
allow_all_external=allow_all_external,
allow_external=allow_external,
allow_unverified=allow_unverified,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
precedence=precedence,
context=context)
self._fetchers.insert(0, fetcher)
# end stub
#####
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher):
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder is not None
def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False, interpreter=None):
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = write_simple_pex(td1, body, dists=dists, coverage=coverage, interpreter=interpreter)
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
return run_simple_pex(pex, args=args, env=env, interpreter=interpreter)
def run_test(body, env=None):
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = write_pex(td1, body)
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
return run_pex(pex, env=env)
