def test_nested_requirements():
with temporary_dir() as td1:
with temporary_dir() as td2:
with open(os.path.join(td1, 'requirements.txt'), 'w') as fp:
fp.write(dedent('''
requirement1
requirement2
-r %s
-r %s
''' % (
os.path.join(td2, 'requirements_nonrelative.txt'),
os.path.join('relative', 'requirements_relative.txt'))
))
with open(os.path.join(td2, 'requirements_nonrelative.txt'), 'w') as fp:
fp.write(dedent('''
requirement3
requirement4
'''))
os.mkdir(os.path.join(td1, 'relative'))
with open(os.path.join(td1, 'relative', 'requirements_relative.txt'), 'w') as fp:
fp.write(dedent('''
requirement5
requirement6
'''))
def rr(req):
return ResolvableRequirement.from_string(req, ResolverOptionsBuilder())
reqs = requirements_from_file(os.path.join(td1, 'requirements.txt'))
assert reqs == [rr('requirement%d' % k) for k in (1, 2, 3, 4, 5, 6)]
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex, interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pb = write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with open_zip(p1.location, 'r') as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
pb = write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_pex_builder_compilation():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, 'src.py')
with open(src, 'w') as fp:
fp.write(exe_main)
exe = os.path.join(td1, 'exe.py')
with open(exe, 'w') as fp:
fp.write(exe_main)
def build_and_check(path, precompile):
pb = PEXBuilder(path=path)
pb.add_source(src, 'lib/src.py')
pb.set_executable(exe, 'exe.py')
pb.freeze(bytecode_compile=precompile)
for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, BOOTSTRAP_DIR)
bootstrap_pycs = []
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
build_and_check(td2, False)
build_and_check(td3, True)
def test_pex_builder_deterministic_timestamp():
pb = PEXBuilder()
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target, deterministic_timestamp=True)
with zipfile.ZipFile(target) as zf:
assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())
def test_iter_pth_paths(mock_exists):
# Ensure path checking always returns True for dummy paths.
mock_exists.return_value = True
with temporary_dir() as tmpdir:
in_tmp = lambda f: os.path.join(tmpdir, f)
PTH_TEST_MAPPING = {
# A mapping of .pth file content -> expected paths.
'/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python\n': [
'/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python'
],
'relative_path\nrelative_path2\n\nrelative_path3': [
in_tmp('relative_path'),
in_tmp('relative_path2'),
in_tmp('relative_path3')
],
'duplicate_path\nduplicate_path': [in_tmp('duplicate_path')],
'randompath\nimport nosuchmodule\n': [in_tmp('randompath')],
'import sys\nfoo\n/bar/baz': [in_tmp('foo'), '/bar/baz'],
'import nosuchmodule\nfoo': [],
'import nosuchmodule\n': [],
'import bad)syntax\n': [],
}
for i, pth_content in enumerate(PTH_TEST_MAPPING):
pth_tmp_path = os.path.abspath(os.path.join(tmpdir, 'test%s.pth' % i))
with open(pth_tmp_path, 'wb') as f:
f.write(to_bytes(pth_content))
assert sorted(PTH_TEST_MAPPING[pth_content]) == sorted(list(iter_pth_paths(pth_tmp_path)))
def test_crawler_local():
FL = ('a.txt', 'b.txt', 'c.txt')
with temporary_dir() as td:
for fn in FL:
with open(os.path.join(td, fn), 'w'):
pass
for dn in (1, 2):
os.mkdir(os.path.join(td, 'dir%d' % dn))
for fn in FL:
with open(os.path.join(td, 'dir%d' % dn, fn), 'w'):
pass
# basic file / dir rel splitting
links, rels = Crawler.crawl_local(Link.wrap(td))
assert set(links) == set(Link.wrap(os.path.join(td, fn)) for fn in FL)
assert set(rels) == set(Link.wrap(os.path.join(td, 'dir%d' % n)) for n in (1, 2))
# recursive crawling, single vs multi-threaded
for caching in (False, True):
for threads in (1, 2, 3):
links = Crawler(threads=threads).crawl([td], follow_links=True)
expect_links = (set(Link.wrap(os.path.join(td, fn)) for fn in FL) |
set(Link.wrap(os.path.join(td, 'dir1', fn)) for fn in FL) |
set(Link.wrap(os.path.join(td, 'dir2', fn)) for fn in FL))
assert set(links) == expect_links
def test_pex_paths():
# Tests that PEX_PATH allows importing sources from the referenced pex.
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex1_path = os.path.join(temp_dir, 'pex1')
write_simple_pex(
pex1_path,
exe_contents='',
sources=[
('foo_pkg/__init__.py', ''),
('foo_pkg/foo_module.py', 'def foo_func():\n return "42"')
]
)
pex2_path = os.path.join(temp_dir, 'pex2')
pex2 = write_simple_pex(
pex2_path,
'import sys; from bar_pkg.bar_module import bar_func; '
'sys.stdout.write(bar_func()); sys.exit(0)',
sources=[
('bar_pkg/bar_module.py',
'from foo_pkg.foo_module import foo_func\ndef bar_func():\n return foo_func()')
]
)
rc = PEX(pex2.path()).run(stdin=None, stdout=fake_stdout, env={'PEX_PATH': pex1_path})
assert rc == 0
fake_stdout.seek(0)
assert fake_stdout.read() == b'42'
def compilation(valid_paths=None, invalid_paths=None, compile_paths=None):
with temporary_dir() as root:
for path in valid_paths:
write_source(os.path.join(root, path))
for path in invalid_paths:
write_source(os.path.join(root, path), valid=False)
compiler = Compiler(PythonInterpreter.get())
yield root, compiler.compile(root, compile_paths)
def test_site_libs():
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages])
site_libs = PEX.site_libs()
assert site_packages in site_libs
def test_executor_execute_dir():
with temporary_dir() as temp_dir:
test_dir = os.path.realpath(os.path.join(temp_dir, 'tmp'))
safe_mkdir(test_dir)
assert os.path.isdir(test_dir)
with pytest.raises(Executor.ExecutionError) as e:
Executor.execute(test_dir)
assert test_dir in str(e)
def bdist_pex(project_dir, bdist_args=None):
with temporary_dir() as dist_dir:
cmd = [sys.executable, 'setup.py', 'bdist_pex', '--bdist-dir={}'.format(dist_dir)]
if bdist_args:
cmd.extend(bdist_args)
subprocess.check_call(cmd, cwd=project_dir)
dists = os.listdir(dist_dir)
assert len(dists) == 1
yield os.path.join(dist_dir, dists[0])
def test_pex_python_symlink():
with temporary_dir() as td:
symlink_path = os.path.join(td, 'python-symlink')
os.symlink(sys.executable, symlink_path)
pexrc_path = os.path.join(td, '.pexrc')
with open(pexrc_path, 'w') as pexrc:
pexrc.write("PEX_PYTHON=%s" % symlink_path)
body = "print('Hello')"
_, rc = run_simple_pex_test(body, coverage=True, env=make_env(HOME=td))
assert rc == 0
def test_site_libs_excludes_prefix():
"""Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages, sys.prefix])
site_libs = PEX.site_libs()
assert site_packages in site_libs
assert sys.prefix not in site_libs
def test_ambiguous_transitive_resolvable():
# If an unbounded or larger bounded resolvable is resolved first, and a
# transitive resolvable is resolved later in another round, Error(Ambiguous resolvable) can be
# raised because foo pulls in foo-2.0.0 and bar->foo==1.0.0 pulls in foo-1.0.0.
foo1_0 = make_sdist(name='foo', version='1.0.0')
foo2_0 = make_sdist(name='foo', version='2.0.0')
bar1_0 = make_sdist(name='bar',
version='1.0.0',
install_reqs=['foo==1.0.0'])
with temporary_dir() as td:
for sdist in (foo1_0, foo2_0, bar1_0):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
resolved_dists = do_resolve_multi(['foo', 'bar'],
fetchers=fetchers,
cache=cd,
cache_ttl=1000)
assert len(resolved_dists) == 2
assert resolved_dists[0].distribution.version == '1.0.0'
def test_pex_executable():
# Tests that pex keeps executable permissions
with temporary_dir() as temp_dir:
pex_dir = os.path.join(temp_dir, 'pex_dir')
safe_mkdir(pex_dir)
with open(os.path.join(pex_dir, 'exe.py'), 'w') as fp:
fp.write(
textwrap.dedent('''
import subprocess
import os
import sys
import my_package
path = os.path.join(os.path.dirname(my_package.__file__), 'bin/start.sh')
sys.stdout.write(subprocess.check_output([path]).decode('utf-8'))
'''))
project_content = {
'setup.py':
textwrap.dedent('''
from setuptools import setup
setup(
name='my_project',
version='0.0.0.0',
zip_safe=True,
packages=['my_package'],
package_data={'my_package': ['bin/*']},
install_requires=[],
)
'''),
'my_package/__init__.py':
0,
'my_package/bin/start.sh': ("#!/usr/bin/env bash\n"
"echo 'hello world from start.sh!'"),
'my_package/my_module.py':
'def do_something():\n print("hello world!")\n',
}
pex_builder = PEXBuilder(path=pex_dir)
with temporary_content(project_content, perms=0o755) as project_dir:
installer = WheelInstaller(project_dir)
bdist = DistributionHelper.distribution_from_path(
installer.bdist())
pex_builder.add_dist_location(bdist.location)
pex_builder.set_executable(os.path.join(pex_dir, 'exe.py'))
pex_builder.freeze()
app_pex = os.path.join(os.path.join(temp_dir, 'out_pex_dir'),
'app.pex')
pex_builder.build(app_pex)
std_out, rc = run_simple_pex(
app_pex, env={'PEX_ROOT': os.path.join(temp_dir, '.pex')})
assert rc == 0
assert std_out.decode('utf-8') == 'hello world from start.sh!\n'
def test_interpreter_constraints_to_pex_info_py2():
with temporary_dir() as output_dir:
# target python 2
pex_out_path = os.path.join(output_dir, 'pex_py2.pex')
res = run_pex_command(['--disable-cache',
'--interpreter-constraint=>=2.7',
'--interpreter-constraint=<3',
'-o', pex_out_path])
res.assert_success()
pex_info = get_pex_info(pex_out_path)
assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints)
def test_issues_661_devendoring_required():
# The cryptography distribution does not have a whl released for python3 on linux at version 2.5.
# As a result, we're forced to build it under python3 and, prior to the fix for
# https://github.com/pantsbuild/pex/issues/661, this would fail using the vendored setuptools
# inside pex.
with temporary_dir() as td:
cryptography_pex = os.path.join(td, 'cryptography.pex')
res = run_pex_command(['cryptography==2.5', '-o', cryptography_pex])
res.assert_success()
subprocess.check_call([cryptography_pex, '-c', 'import cryptography'])
def test_pex_run_custom_setuptools_useable():
resolved_dists = resolve(['setuptools==36.2.7'])
dists = [resolved_dist.distribution for resolved_dist in resolved_dists]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from setuptools.sandbox import run_setup',
dists=dists,
)
rc = PEX(pex.path()).run()
assert rc == 0
def test_setup_interpreter_constraint():
interpreter = ensure_python_interpreter(PY27)
with temporary_dir() as out:
pex = os.path.join(out, 'pex.pex')
results = run_pex_command(['jsonschema==2.6.0',
'--disable-cache',
'--interpreter-constraint=CPython=={}'.format(PY27),
'-o', pex],
env=make_env(PATH=os.path.dirname(interpreter)))
results.assert_success()
subprocess.check_call([pex, '-c', 'import jsonschema'])
def test_load_internal_cache_unzipped():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True),
temporary_dir()) as (pb, pex_root):
pb.info.pex_root = pex_root
pb.freeze()
dists = list(PEXEnvironment._load_internal_cache(pb.path(), pb.info))
assert len(dists) == 1
assert normalize(dists[0].location).startswith(
normalize(os.path.join(pb.path(), pb.info.internal_cache)))
def test_site_libs_excludes_prefix():
"""Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""
with nested(mock.patch.object(PEX, '_get_site_packages'),
temporary_dir()) as (mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
mock_site_packages.return_value = set([site_packages, sys.prefix])
site_libs = PEX.site_libs()
assert site_packages in site_libs
assert sys.prefix not in site_libs
def test_resolve_prereleases_cached():
stable_dep = make_sdist(name='dep', version='2.0.0')
prerelease_dep = make_sdist(name='dep', version='3.0.0rc3')
with temporary_dir() as td:
for sdist in (stable_dep, prerelease_dep):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
def assert_resolve(dep, expected_version, **resolve_kwargs):
resolved_dists = do_resolve_multi([dep],
cache=cd,
cache_ttl=1000,
**resolve_kwargs)
assert 1 == len(resolved_dists)
resolved_dist = resolved_dists[0]
assert expected_version == resolved_dist.distribution.version
Crawler.reset_cache()
# First do a run to load it into the cache.
assert_resolve('dep>=1,<4',
'3.0.0rc3',
allow_prereleases=True,
fetchers=fetchers)
# This simulates running from another pex command. The Crawler cache actually caches an empty
# cache so this fails in the same "process".
Crawler.reset_cache()
# Now assert that we can get it from the cache by removing the source.
assert_resolve('dep>=1,<4',
'3.0.0rc3',
allow_prereleases=True,
fetchers=[])
# It should also be able to resolve without allow_prereleases, if explicitly requested.
Crawler.reset_cache()
assert_resolve('dep>=1.rc1,<4', '3.0.0rc3', fetchers=[])
def bdist_pex(project_dir, bdist_args=None):
with temporary_dir() as dist_dir:
cmd = [
sys.executable, 'setup.py', 'bdist_pex',
'--bdist-dir={}'.format(dist_dir)
]
if bdist_args:
cmd.extend(bdist_args)
subprocess.check_call(cmd, cwd=project_dir)
dists = os.listdir(dist_dir)
assert len(dists) == 1
yield os.path.join(dist_dir, dists[0])
def test_pex_source_bundling():
with temporary_dir() as output_dir:
with temporary_dir() as input_dir:
with open(os.path.join(input_dir, 'exe.py'), 'w') as fh:
fh.write(dedent('''
print('hello')
'''
))
pex_path = os.path.join(output_dir, 'pex1.pex')
res = run_pex_command([
'-o', pex_path,
'-D', input_dir,
'-e', 'exe',
])
res.assert_success()
stdout, rc = run_simple_pex(pex_path)
assert rc == 0
assert stdout == b'hello\n'
def test_platform_specific_egg_resolution():
with temporary_dir() as td:
pex_out_path = os.path.join(td, 'pex.pex')
res = run_pex_command(['--disable-cache',
'--no-wheel',
'--no-build',
'--no-pypi',
'--platform=linux-x86_64',
'--find-links=tests/example_packages/',
'M2Crypto==0.22.3',
'-o', pex_out_path])
res.assert_success()
def test_site_libs_symlink():
with nested(mock.patch.object(PEX, '_get_site_packages'),
temporary_dir()) as (mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
site_packages_link = os.path.join(tempdir, 'site-packages-link')
os.symlink(site_packages, site_packages_link)
mock_site_packages.return_value = set([site_packages_link])
site_libs = PEX.site_libs()
assert os.path.realpath(site_packages) in site_libs
assert site_packages_link in site_libs
def test_site_libs_symlink():
with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
mock_site_packages, tempdir):
site_packages = os.path.join(tempdir, 'site-packages')
os.mkdir(site_packages)
site_packages_link = os.path.join(tempdir, 'site-packages-link')
os.symlink(site_packages, site_packages_link)
mock_site_packages.return_value = set([site_packages_link])
site_libs = PEX.site_libs()
assert os.path.realpath(site_packages) in site_libs
assert site_packages_link in site_libs
def test_pex_run():
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'import sys; sys.stdout.write("hello"); sys.stderr.write("hello"); sys.exit(0)'
)
rc = PEX(pex.path()).run(stdin=None, stdout=fake_stdout, stderr=fake_stdout)
assert rc == 0
fake_stdout.seek(0)
assert fake_stdout.read() == b'hellohello'
def test_pex_run():
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'import sys; sys.stdout.write("hello"); sys.stderr.write("hello"); sys.exit(0)'
)
rc = PEX(pex.path()).run(stdin=None, stdout=fake_stdout, stderr=fake_stdout)
assert rc == 0
fake_stdout.seek(0)
assert fake_stdout.read() == b'hellohello'
def test_platform_specific_egg_resolution_matching():
with temporary_dir() as td:
pex_out_path = os.path.join(td, 'pex.pex')
res = run_pex_command([
'--disable-cache',
'--no-wheel',
'--no-build',
'netifaces==0.10.6', # Only provides win32 eggs.
'-o',
pex_out_path
])
res.assert_failure()
def yield_pex_builder(zip_safe=True,
installer_impl=EggInstaller,
interpreter=None):
with nested(
temporary_dir(),
make_bdist('p1',
zipped=True,
zip_safe=zip_safe,
installer_impl=installer_impl,
interpreter=interpreter)) as (td, p1):
pb = PEXBuilder(path=td, interpreter=interpreter)
pb.add_dist_location(p1.location)
yield pb
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_execute_interpreter_dashc_program():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.freeze()
process = PEX(pex_chroot).run(args=['-c', 'import sys; print(" ".join(sys.argv))', 'one'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b'-c one\n' == stdout
assert b'' == stderr
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_resolve_extra_sdist():
project1_sdist = make_sdist(name='project1',
version='1.0.0',
extras_require={'foo': ['project2']})
project2_sdist = make_sdist(name='project2', version='2.0.0')
with temporary_dir() as td:
for sdist in (project1_sdist, project2_sdist):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
resolved_dists = do_resolve_multi(['project1[foo]'], fetchers=fetchers)
assert ({_parse_requirement(req) for req in ('project1[foo]', 'project2; extra=="foo"')} ==
{_parse_requirement(resolved_dist.requirement) for resolved_dist in resolved_dists})
def test_cached_dependency_pinned_unpinned_resolution_multi_run():
# This exercises the issue described here: https://github.com/pantsbuild/pex/issues/178
project1_0_0 = make_sdist(name='project', version='1.0.0')
project1_1_0 = make_sdist(name='project', version='1.1.0')
with temporary_dir() as td:
for sdist in (project1_0_0, project1_1_0):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
# First run, pinning 1.0.0 in the cache
resolved_dists = do_resolve_multi(['project', 'project==1.0.0'],
fetchers=fetchers,
cache=cd,
cache_ttl=1000)
assert len(resolved_dists) == 1
assert resolved_dists[0].distribution.version == '1.0.0'
# This simulates separate invocations of pex but allows us to keep the same tmp cache dir
Crawler.reset_cache()
# Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also
# return SourcePackages found in td
resolved_dists = do_resolve_multi(['project', 'project==1.1.0'],
fetchers=fetchers,
cache=cd,
cache_ttl=1000)
assert len(resolved_dists) == 1
assert resolved_dists[0].distribution.version == '1.1.0'
# Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact
# resolvable should pull from pypi as well since inexact will and the resulting
# resolvable_set.merge() would fail.
Crawler.reset_cache()
time.sleep(1)
resolved_dists = do_resolve_multi(['project', 'project==1.1.0'],
fetchers=fetchers,
cache=cd,
cache_ttl=1)
assert len(resolved_dists) == 1
assert resolved_dists[0].distribution.version == '1.1.0'
def test_force_local():
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment.force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py', '__main__.pyc'])
# idempotence
assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_interpreter_constraints_to_pex_info_py3():
py3_interpreter = ensure_python_interpreter(PY36)
with temporary_dir() as output_dir:
# target python 3
pex_out_path = os.path.join(output_dir, 'pex_py3.pex')
res = run_pex_command(
[
'--disable-cache', '--interpreter-constraint=>3', '-o',
pex_out_path
],
env=make_env(PATH=os.path.dirname(py3_interpreter)))
res.assert_success()
pex_info = get_pex_info(pex_out_path)
assert ['>3'] == pex_info.interpreter_constraints
def test_pex_builder_shebang():
def builder(shebang):
pb = PEXBuilder()
pb.set_shebang(shebang)
return pb
for pb in builder('foobar'), builder('#!foobar'):
for b in pb, pb.clone():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
b.build(target)
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
def test_multiplatform_entrypoint():
with temporary_dir() as td:
pex_out_path = os.path.join(td, 'p537.pex')
interpreter = ensure_python_interpreter(PY36)
res = run_pex_command([
'p537==1.0.3', '--no-build', '--python={}'.format(interpreter),
'--python-shebang=#!{}'.format(interpreter),
'--platform=linux-x86_64', '--platform=macosx-10.13-x86_64', '-c',
'p537', '-o', pex_out_path, '--validate-entry-point'
])
res.assert_success()
greeting = subprocess.check_output([pex_out_path])
assert b'Hello World!' == greeting.strip()
def test_pex_path_in_pex_info_and_env():
with temporary_dir() as output_dir:
# create 2 pex files for PEX-INFO pex_path
pex1_path = os.path.join(output_dir, 'pex1.pex')
res1 = run_pex_command(
['--disable-cache', 'requests', '-o', pex1_path])
res1.assert_success()
pex2_path = os.path.join(output_dir, 'pex2.pex')
res2 = run_pex_command(['--disable-cache', 'flask', '-o', pex2_path])
res2.assert_success()
pex_path = ':'.join(
os.path.join(output_dir, name)
for name in ('pex1.pex', 'pex2.pex'))
# create a pex for environment PEX_PATH
pex3_path = os.path.join(output_dir, 'pex3.pex')
res3 = run_pex_command(['--disable-cache', 'wheel', '-o', pex3_path])
res3.assert_success()
env_pex_path = os.path.join(output_dir, 'pex3.pex')
# parameterize the pex arg for test.py
pex_out_path = os.path.join(output_dir, 'out.pex')
# create test file test.py that attempts to import modules from pex1/pex2
test_file_path = os.path.join(output_dir, 'test.py')
with open(test_file_path, 'w') as fh:
fh.write(
dedent('''
import requests
import flask
import wheel
import sys
import os
import subprocess
print('Success!')
'''))
# build out.pex composed from pex1/pex1
run_pex_command([
'--disable-cache', '--pex-path={}'.format(pex_path), '-o',
pex_out_path
])
# load secondary PEX_PATH
env = make_env(PEX_PATH=env_pex_path)
# run test.py with composite env
stdout, rc = run_simple_pex(pex_out_path, [test_file_path], env=env)
assert rc == 0
assert stdout == b'Success!\n'
def test_pex_repl_built():
"""Tests the REPL in the context of a built pex."""
stdin_payload = b'import requests; import sys; sys.exit(3)'
with temporary_dir() as output_dir:
# Create a temporary pex containing just `requests` with no entrypoint.
pex_path = os.path.join(output_dir, 'requests.pex')
results = run_pex_command(['--disable-cache', 'requests', '-o', pex_path])
results.assert_success()
# Test that the REPL is functional.
stdout, rc = run_simple_pex(pex_path, stdin=stdin_payload)
assert rc == 3
assert b'>>>' in stdout
def test_activate_interpreter_different_from_current():
with temporary_dir() as pex_root:
interp_version = PY36 if PY2 else PY27
custom_interpreter = PythonInterpreter.from_binary(
ensure_python_interpreter(interp_version))
pex_info = PexInfo.default(custom_interpreter)
pex_info.pex_root = pex_root
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot,
interpreter=custom_interpreter,
pex_info=pex_info)
with make_bdist(installer_impl=WheelInstaller,
interpreter=custom_interpreter) as bdist:
pex_builder.add_distribution(bdist)
pex_builder.set_entry_point('sys:exit')
pex_builder.freeze()
pex = PEX(pex_builder.path(), interpreter=custom_interpreter)
try:
pex._activate()
except SystemExit as e:
pytest.fail('PEX activation of %s failed with %s' %
(pex, e))
def test_pex_builder_shebang():
def builder(shebang):
pb = PEXBuilder()
pb.set_shebang(shebang)
return pb
for pb in builder('foobar'), builder('#!foobar'):
for b in pb, pb.clone():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
b.build(target)
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
def _add_test_hello_to_pex(ep):
with temporary_dir() as td:
hello_file = "\n".join([
"def hello():",
" print('hello')",
])
with temporary_file(root_dir=td) as tf:
with open(tf.name, 'w') as handle:
handle.write(hello_file)
pex_builder = PEXBuilder()
pex_builder.add_source(tf.name, 'test.py')
pex_builder.set_entry_point(ep)
pex_builder.freeze()
yield pex_builder
def test_pex_run_conflicting_custom_setuptools_useable():
# Here we use an older setuptools to build the pex which has a newer setuptools requirement.
# These setuptools dists have different pkg_resources APIs:
# $ diff \
# <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \
# <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort)
# 2a3,4
# > pkg_resources/py31compat.py
# > pkg_resources/_vendor/appdirs.py
with temporary_dir() as resolve_cache:
dists = resolve(['setuptools==20.3.1'], cache=resolve_cache)
interpreter = PythonInterpreter.from_binary(
sys.executable,
path_extras=[dist.location for dist in dists],
include_site_extras=False)
dists = resolve(['setuptools==40.4.3'], cache=resolve_cache)
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from pkg_resources import appdirs, py31compat',
dists=dists,
interpreter=interpreter)
rc = PEX(pex.path()).run()
assert rc == 0
def assert_chroot_perms(copyfn):
with temporary_dir() as src:
one = os.path.join(src, 'one')
touch(one)
two = os.path.join(src, 'two')
touch(two)
chmod_plus_x(two)
with temporary_dir() as dst:
chroot = Chroot(dst)
copyfn(chroot, one, 'one')
copyfn(chroot, two, 'two')
assert extract_perms(one) == extract_perms(os.path.join(chroot.path(), 'one'))
assert extract_perms(two) == extract_perms(os.path.join(chroot.path(), 'two'))
zip_path = os.path.join(src, 'chroot.zip')
chroot.zip(zip_path)
with temporary_dir() as extract_dir:
with contextlib.closing(PermPreservingZipFile(zip_path)) as zf:
zf.extractall(extract_dir)
assert extract_perms(one) == extract_perms(os.path.join(extract_dir, 'one'))
assert extract_perms(two) == extract_perms(os.path.join(extract_dir, 'two'))
def test_pex_builder_copy_or_link():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, 'exe.py')
with open(src, 'w') as fp:
fp.write(exe_main)
def build_and_check(path, copy):
pb = PEXBuilder(path=path, copy=copy)
pb.add_source(src, 'exe.py')
path_clone = os.path.join(path, '__clone')
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
build_and_check(td2, False)
build_and_check(td3, True)
def zip_fixture():
with temporary_dir() as target_dir:
one = os.path.join(target_dir, 'one')
touch(one)
two = os.path.join(target_dir, 'two')
touch(two)
chmod_plus_x(two)
assert extract_perms(one) != extract_perms(two)
zip_file = os.path.join(target_dir, 'test.zip')
with contextlib.closing(PermPreservingZipFile(zip_file, 'w')) as zf:
zf.write(one, 'one')
zf.write(two, 'two')
yield zip_file, os.path.join(target_dir, 'extract'), one, two
def write_and_run_simple_pex(inheriting=False):
"""Write a pex file that contains an executable entry point
:param inheriting: whether this pex should inherit site-packages paths
:type inheriting: bool
"""
with temporary_dir() as td:
pex_path = os.path.join(td, 'show_path.pex')
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write('') # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
pb.build(pex_path)
yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0]
def test_pex_builder_preamble():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
should_create = os.path.join(td, 'foo.1')
tempfile_preamble = "\n".join([
"import sys",
"open('{0}', 'w').close()".format(should_create),
"sys.exit(3)"
])
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def test_clp_prereleases_resolver():
prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
with nested(temporary_dir(), temporary_dir()) as (dist_dir, cache_dir):
safe_copy(prerelease_dep, os.path.join(dist_dir, os.path.basename(prerelease_dep)))
fetcher = Fetcher([dist_dir])
# When no specific options are specified, allow_prereleases is None
parser, resolver_options_builder = configure_clp()
assert resolver_options_builder._allow_prereleases is None
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(args=['--cache-dir', cache_dir, # Avoid dangling {pex_root}.
'--pre', 'dep==1.2.3b1', 'dep'])
assert resolver_options_builder._allow_prereleases
# We need to use our own fetcher instead of PyPI
resolver_options_builder._fetchers.insert(0, fetcher)
#####
# The resolver created during processing of command line options (configure_clp)
# is not actually passed into the API call (resolve_multi) from build_pex().
# Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
# is created. The only way to supply our own fetcher to that new instance is to patch it
# here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
# below and the change in the `pex.resolver` module where the patched object resides.
#
import pex.resolver
class BuilderWithFetcher(ResolverOptionsBuilder):
def __init__(self,
fetchers=None,
allow_all_external=False,
allow_external=None,
allow_unverified=None,
allow_prereleases=None,
use_manylinux=None,
precedence=None,
context=None
):
super(BuilderWithFetcher, self).__init__(fetchers=fetchers,
allow_all_external=allow_all_external,
allow_external=allow_external,
allow_unverified=allow_unverified,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
precedence=precedence,
context=context)
self._fetchers.insert(0, fetcher)
# end stub
#####
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher):
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder is not None
