def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(),
temporary_filename()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file, pb.info)
assert len(zip_safe) == 1
assert normalize(zip_safe[0].location).startswith(
normalize(os.path.join(pex_file, pb.info.internal_cache))), (
'loc: %s, cache: %s' %
(normalize(zip_safe[0].location),
normalize(os.path.join(pex_file, pb.info.internal_cache))))
pb.info.always_write_cache = True
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(
normalize(pb.info.install_cache))
# Check that we can read from the cache
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location).startswith(
normalize(pb.info.install_cache))
# non-zip_safe pex will be written to install cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(),
temporary_filename()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(
normalize(pb.info.install_cache))
original_location = normalize(new[0].location)
# do the second time to validate idempotence of caching
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
pex_file, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location) == original_location
def test_osx_platform_intel_issue_523():
def bad_interpreter():
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
# setuptools and wheel extras.
with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=pb.interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
def run(args, **env):
pex_env = os.environ.copy()
pex_env['PEX_VERBOSE'] = '1'
pex_env.update(**env)
process = pex.run(args=args,
env=pex_env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
return process.returncode, stdout, stderr
returncode, _, stderr = run(['-c', 'import psutil'])
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
returncode, stdout, stderr = run(['-c', 'import pkg_resources'])
assert 0 != returncode, (
'Isolated pex process succeeded but should not have found pkg-resources:\n'
'STDOUT:\n'
'{}\n'
'STDERR:\n'
'{}'
.format(stdout, stdout, stderr)
)
returncode, stdout, stderr = run(
['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'],
# Let the bad interpreter site-packages setuptools leak in.
PEX_INHERIT_PATH='1'
)
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
# Verify this worked along side the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_force_local():
# type: () -> None
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
pb,
pex_root,
pex_file,
):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment._force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
listing = set(os.listdir(pb.info.zip_unsafe_cache))
# The code_cache should be a write-locked directory.
assert len(listing) == 2
listing.remove(os.path.basename(code_cache))
lockfile = listing.pop()
assert os.path.isfile(os.path.join(pb.info.zip_unsafe_cache, lockfile))
assert set(os.listdir(code_cache)) == {PexInfo.PATH, "__main__.py", "__main__.pyc"}
# idempotence
assert PEXEnvironment._force_local(pex_file, pb.info) == code_cache
def assert_dist_cache(zip_safe):
# type: (bool) -> None
with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(),
temporary_filename()) as (
pb,
pex_root,
pex_file,
):
pb.info.pex_root = pex_root
pb.build(pex_file)
with open_zip(pex_file) as zf:
dists = PEXEnvironment._write_zipped_internal_cache(
zf=zf, pex_info=pb.info)
assert len(dists) == 1
original_location = normalize(dists[0].location)
assert original_location.startswith(
normalize(pb.info.install_cache))
# Call a second time to validate idempotence of caching.
dists = PEXEnvironment._write_zipped_internal_cache(zf=None,
pex_info=pb.info)
assert len(dists) == 1
assert normalize(dists[0].location) == original_location
def test_osx_platform_intel_issue_523():
def bad_interpreter(include_site_extras=True):
return PythonInterpreter.from_binary(
_KNOWN_BAD_APPLE_INTERPRETER,
include_site_extras=include_site_extras)
interpreter = bad_interpreter(include_site_extras=False)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for resolved_dist in resolver.resolve(
[requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version,
dist.location)
with nested(
yield_pex_builder(installer_impl=WheelInstaller,
interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage,
WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
args = [
'-c',
'import pkg_resources; print(pkg_resources.get_supported_platform())'
]
env = os.environ.copy()
env['PEX_VERBOSE'] = '1'
process = pex.run(args=args,
env=env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(
process.returncode, stderr))
# Verify this all worked under the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes(
'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_write_zipped_internal_cache():
# zip_safe pex will not be written to install cache unless always_write_cache
with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_filename()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
assert len(zip_safe) == 1
assert normalize(zip_safe[0].location).startswith(
normalize(os.path.join(pex_file, pb.info.internal_cache))), (
'loc: %s, cache: %s' % (
normalize(zip_safe[0].location),
normalize(os.path.join(pex_file, pb.info.internal_cache))))
pb.info.always_write_cache = True
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
# Check that we can read from the cache
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location).startswith(normalize(pb.info.install_cache))
# non-zip_safe pex will be written to install cache
with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_filename()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
assert len(new) == 1
assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
original_location = normalize(new[0].location)
# do the second time to validate idempotence of caching
existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info)
assert len(existing) == 1
assert normalize(existing[0].location) == original_location
def test_force_local():
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment.force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
assert set(os.listdir(code_cache)) == set([PexInfo.PATH, "__main__.py", "__main__.pyc"])
# idempotence
assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_force_local():
with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
pb, pex_root, pex_file):
pb.info.pex_root = pex_root
pb.build(pex_file)
code_cache = PEXEnvironment.force_local(pex_file, pb.info)
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py', '__main__.pyc'])
# idempotence
assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_osx_platform_intel_issue_523():
# type: () -> None
def bad_interpreter():
# type: () -> PythonInterpreter
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
# setuptools and wheel extras.
with yield_pex_builder(
interpreter=bad_interpreter()
) as pb, temporary_filename() as pex_file:
for resolved_dist in resolver.resolve(
["psutil==5.4.3"], cache=cache, interpreter=pb.interpreter
):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
def run(args, **env):
# type: (Iterable[str], **str) -> Tuple[int, str, str]
pex_env = os.environ.copy()
pex_env["PEX_VERBOSE"] = "1"
pex_env.update(**env)
process = pex.run(
args=args,
env=pex_env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
return process.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")
returncode, _, stderr = run(["-c", "import psutil"])
assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format(
returncode, stderr
)
returncode, stdout, stderr = run(["-c", "import pkg_resources"])
assert 0 != returncode, (
"Isolated pex process succeeded but should not have found pkg-resources:\n"
"STDOUT:\n"
"{}\n"
"STDERR:\n"
"{}".format(stdout, stderr)
)
returncode, stdout, stderr = run(
["-c", "import pkg_resources; print(pkg_resources.get_supported_platform())"],
# Let the bad interpreter site-packages setuptools leak in.
PEX_INHERIT_PATH=InheritPath.for_value(True).value,
)
assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format(
returncode, stderr
)
# Verify this worked along side the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = ".".join(release.split(".")[:2])
assert "macosx-{}-intel".format(major_minor) == stdout.strip()
