def test_write_zipped_internal_cache(): # zip_safe pex will not be written to install cache unless always_write_cache with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache( pex_file, pb.info) assert len(zip_safe) == 1 assert normalize(zip_safe[0].location).startswith( normalize(os.path.join(pex_file, pb.info.internal_cache))), ( 'loc: %s, cache: %s' % (normalize(zip_safe[0].location), normalize(os.path.join(pex_file, pb.info.internal_cache)))) pb.info.always_write_cache = True existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache( pex_file, pb.info) assert len(new) == 1 assert normalize(new[0].location).startswith( normalize(pb.info.install_cache)) # Check that we can read from the cache existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache( pex_file, pb.info) assert len(existing) == 1 assert normalize(existing[0].location).startswith( normalize(pb.info.install_cache)) # non-zip_safe pex will be written to install cache with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache( pex_file, pb.info) assert len(new) == 1 assert normalize(new[0].location).startswith( normalize(pb.info.install_cache)) original_location = normalize(new[0].location) # do the second time to validate idempotence of caching existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache( pex_file, pb.info) assert len(existing) == 1 assert normalize(existing[0].location) == original_location
def test_osx_platform_intel_issue_523(): def bad_interpreter(): return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored # setuptools and wheel extras. with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, precedence=(SourcePackage, WheelPackage), interpreter=pb.interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) def run(args, **env): pex_env = os.environ.copy() pex_env['PEX_VERBOSE'] = '1' pex_env.update(**env) process = pex.run(args=args, env=pex_env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() return process.returncode, stdout, stderr returncode, _, stderr = run(['-c', 'import psutil']) assert 0 == returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr) ) returncode, stdout, stderr = run(['-c', 'import pkg_resources']) assert 0 != returncode, ( 'Isolated pex process succeeded but should not have found pkg-resources:\n' 'STDOUT:\n' '{}\n' 'STDERR:\n' '{}' .format(stdout, stdout, stderr) ) returncode, stdout, stderr = run( ['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'], # Let the bad interpreter site-packages setuptools leak in. PEX_INHERIT_PATH='1' ) assert 0 == returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr) ) # Verify this worked along side the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = '.'.join(release.split('.')[:2]) assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_force_local(): # type: () -> None with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file, ): pb.info.pex_root = pex_root pb.build(pex_file) code_cache = PEXEnvironment._force_local(pex_file, pb.info) assert os.path.exists(pb.info.zip_unsafe_cache) listing = set(os.listdir(pb.info.zip_unsafe_cache)) # The code_cache should be a write-locked directory. assert len(listing) == 2 listing.remove(os.path.basename(code_cache)) lockfile = listing.pop() assert os.path.isfile(os.path.join(pb.info.zip_unsafe_cache, lockfile)) assert set(os.listdir(code_cache)) == {PexInfo.PATH, "__main__.py", "__main__.pyc"} # idempotence assert PEXEnvironment._force_local(pex_file, pb.info) == code_cache
def assert_dist_cache(zip_safe): # type: (bool) -> None with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file, ): pb.info.pex_root = pex_root pb.build(pex_file) with open_zip(pex_file) as zf: dists = PEXEnvironment._write_zipped_internal_cache( zf=zf, pex_info=pb.info) assert len(dists) == 1 original_location = normalize(dists[0].location) assert original_location.startswith( normalize(pb.info.install_cache)) # Call a second time to validate idempotence of caching. dists = PEXEnvironment._write_zipped_internal_cache(zf=None, pex_info=pb.info) assert len(dists) == 1 assert normalize(dists[0].location) == original_location
def test_osx_platform_intel_issue_523(): def bad_interpreter(include_site_extras=True): return PythonInterpreter.from_binary( _KNOWN_BAD_APPLE_INTERPRETER, include_site_extras=include_site_extras) interpreter = bad_interpreter(include_site_extras=False) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil. for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): for resolved_dist in resolver.resolve( [requirement], cache=cache, # We can't use wheels since we're bootstrapping them. precedence=(SourcePackage, EggPackage), interpreter=interpreter): dist = resolved_dist.distribution interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) with nested( yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, precedence=(SourcePackage, WheelPackage), interpreter=interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) args = [ '-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())' ] env = os.environ.copy() env['PEX_VERBOSE'] = '1' process = pex.run(args=args, env=env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert 0 == process.returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format( process.returncode, stderr)) # Verify this all worked under the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = '.'.join(release.split('.')[:2]) assert to_bytes( 'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_write_zipped_internal_cache(): # zip_safe pex will not be written to install cache unless always_write_cache with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info) assert len(zip_safe) == 1 assert normalize(zip_safe[0].location).startswith( normalize(os.path.join(pex_file, pb.info.internal_cache))), ( 'loc: %s, cache: %s' % ( normalize(zip_safe[0].location), normalize(os.path.join(pex_file, pb.info.internal_cache)))) pb.info.always_write_cache = True existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info) assert len(new) == 1 assert normalize(new[0].location).startswith(normalize(pb.info.install_cache)) # Check that we can read from the cache existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info) assert len(existing) == 1 assert normalize(existing[0].location).startswith(normalize(pb.info.install_cache)) # non-zip_safe pex will be written to install cache with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info) assert len(new) == 1 assert normalize(new[0].location).startswith(normalize(pb.info.install_cache)) original_location = normalize(new[0].location) # do the second time to validate idempotence of caching existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file, pb.info) assert len(existing) == 1 assert normalize(existing[0].location) == original_location
def test_force_local(): with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) code_cache = PEXEnvironment.force_local(pex_file, pb.info) assert os.path.exists(pb.info.zip_unsafe_cache) assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1 assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache) assert set(os.listdir(code_cache)) == set([PexInfo.PATH, "__main__.py", "__main__.pyc"]) # idempotence assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_force_local(): with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as ( pb, pex_root, pex_file): pb.info.pex_root = pex_root pb.build(pex_file) code_cache = PEXEnvironment.force_local(pex_file, pb.info) assert os.path.exists(pb.info.zip_unsafe_cache) assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1 assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache) assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py', '__main__.pyc']) # idempotence assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
def test_osx_platform_intel_issue_523(): # type: () -> None def bad_interpreter(): # type: () -> PythonInterpreter return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored # setuptools and wheel extras. with yield_pex_builder( interpreter=bad_interpreter() ) as pb, temporary_filename() as pex_file: for resolved_dist in resolver.resolve( ["psutil==5.4.3"], cache=cache, interpreter=pb.interpreter ): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) def run(args, **env): # type: (Iterable[str], **str) -> Tuple[int, str, str] pex_env = os.environ.copy() pex_env["PEX_VERBOSE"] = "1" pex_env.update(**env) process = pex.run( args=args, env=pex_env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() return process.returncode, stdout.decode("utf-8"), stderr.decode("utf-8") returncode, _, stderr = run(["-c", "import psutil"]) assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format( returncode, stderr ) returncode, stdout, stderr = run(["-c", "import pkg_resources"]) assert 0 != returncode, ( "Isolated pex process succeeded but should not have found pkg-resources:\n" "STDOUT:\n" "{}\n" "STDERR:\n" "{}".format(stdout, stderr) ) returncode, stdout, stderr = run( ["-c", "import pkg_resources; print(pkg_resources.get_supported_platform())"], # Let the bad interpreter site-packages setuptools leak in. PEX_INHERIT_PATH=InheritPath.for_value(True).value, ) assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format( returncode, stderr ) # Verify this worked along side the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = ".".join(release.split(".")[:2]) assert "macosx-{}-intel".format(major_minor) == stdout.strip()