def test_clp_preamble_file(): with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes('print "foo!"')) tmpfile.flush() parser, resolver_options_builder = configure_clp() options, reqs = parser.parse_args(args=['--preamble-file', tmpfile.name]) assert options.preamble_file == tmpfile.name pex_builder = build_pex(reqs, options, resolver_options_builder) assert pex_builder._preamble == to_bytes('print "foo!"')
def test_pex_sys_exit_prints_non_numeric_value_no_traceback(): text = 'something went wrong' sys_exit_arg = '"' + text + '"' # encode the string somehow that's compatible with 2 and 3 expected_output = to_bytes(text) + b'\n' _test_sys_exit(sys_exit_arg, expected_output, 1)
def test_iter_pth_paths(mock_exists): # Ensure path checking always returns True for dummy paths. mock_exists.return_value = True with temporary_dir() as tmpdir: in_tmp = lambda f: os.path.join(tmpdir, f) PTH_TEST_MAPPING = { # A mapping of .pth file content -> expected paths. '/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python\n': [ '/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python' ], 'relative_path\nrelative_path2\n\nrelative_path3': [ in_tmp('relative_path'), in_tmp('relative_path2'), in_tmp('relative_path3') ], 'duplicate_path\nduplicate_path': [in_tmp('duplicate_path')], 'randompath\nimport nosuchmodule\n': [in_tmp('randompath')], 'import nosuchmodule\nfoo': [], 'import nosuchmodule\n': [], 'import bad)syntax\n': [], } for i, pth_content in enumerate(PTH_TEST_MAPPING): pth_tmp_path = os.path.abspath(os.path.join(tmpdir, 'test%s.pth' % i)) with open(pth_tmp_path, 'wb') as f: f.write(to_bytes(pth_content)) assert sorted(PTH_TEST_MAPPING[pth_content]) == sorted(list(iter_pth_paths(pth_tmp_path)))
def build(self, filename, bytecode_compile=True, deterministic_timestamp=False): """Package the PEX into a zipfile. :param filename: The filename where the PEX should be stored. :param bytecode_compile: If True, precompile .py files into .pyc files. :param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps. If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the PEXBuilder immutable. """ if not self._frozen: self.freeze(bytecode_compile=bytecode_compile) try: os.unlink(filename + '~') self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~')) except OSError: # The expectation is that the file does not exist, so continue pass if os.path.dirname(filename): safe_mkdir(os.path.dirname(filename)) with open(filename + '~', 'ab') as pexfile: assert os.path.getsize(pexfile.name) == 0 pexfile.write(to_bytes('%s\n' % self._shebang)) self._chroot.zip(filename + '~', mode='a', deterministic_timestamp=deterministic_timestamp) if os.path.exists(filename): os.unlink(filename) os.rename(filename + '~', filename) chmod_plus_x(filename)
def convert(input): if isinstance(input, dict): out = dict() for key, value in input.items(): out[convert(key)] = convert(value) return out elif isinstance(input, list): return [convert(element) for element in input] elif isinstance(input, string): return to_bytes(input) else: return input
def test_to_bytes(): assert isinstance(to_bytes(''), bytes) assert isinstance(to_bytes('abc'), bytes) assert isinstance(to_bytes(b'abc'), bytes) assert isinstance(to_bytes(u'abc'), bytes) assert isinstance(to_bytes(b'abc'.decode('latin-1'), encoding='utf-8'), bytes) for bad_values in (123, None): with pytest.raises(ValueError): to_bytes(bad_values)
def test_clp_preamble_file(): # type: () -> None with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes('print "foo!"')) tmpfile.flush() parser = configure_clp() options = parser.parse_args(args=["--preamble-file", tmpfile.name]) assert options.preamble_file == tmpfile.name pex_builder = build_pex(options.requirements, options) assert pex_builder._preamble == 'print "foo!"'
def test_clp_preamble_file(): with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes('print "foo!"')) tmpfile.flush() parser = configure_clp() options, reqs = parser.parse_args( args=['--preamble-file', tmpfile.name]) assert options.preamble_file == tmpfile.name pex_builder = build_pex(reqs, options) assert pex_builder._preamble == 'print "foo!"'
def test_to_bytes(): assert isinstance(to_bytes(''), bytes) assert isinstance(to_bytes('abc'), bytes) assert isinstance(to_bytes(b'abc'), bytes) assert isinstance(to_bytes(u'abc'), bytes) assert isinstance(to_bytes(b'abc'.decode('latin-1'), encoding='utf-8'), bytes) for bad_value in (123, None): with pytest.raises(ValueError): to_bytes(bad_value)
def test_get_script_from_whl(): whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl' dists = list(find_wheels_in_zip(zipimport.zipimporter(whl_path), whl_path)) assert len(dists) == 1 dist = dists[0] assert 'aws-cfn-bootstrap' == dist.project_name script_path, script_content = get_script_from_whl('cfn-signal', dist) assert os.path.join(whl_path, 'aws_cfn_bootstrap-1.4.data/scripts/cfn-signal') == script_path assert script_content.startswith(to_bytes('#!')), 'Expected a `scripts`-style script w/shebang.' assert (None, None) == get_script_from_whl('non_existent_script', dist)
def test_to_bytes(): assert isinstance(to_bytes(""), bytes) assert isinstance(to_bytes("abc"), bytes) assert isinstance(to_bytes(b"abc"), bytes) assert isinstance(to_bytes(u"abc"), bytes) assert isinstance(to_bytes(b"abc".decode("latin-1"), encoding="utf-8"), bytes) for bad_value in (123, None): with pytest.raises(ValueError): to_bytes(bad_value)
def _get_external_dependencies(self, binary_target): artifacts_by_file_name = defaultdict(set) for external_dep, coordinate in self.list_external_jar_dependencies(binary_target): self.context.log.debug(' scanning {} from {}'.format(coordinate, external_dep)) with open_zip(external_dep) as dep_zip: for qualified_file_name in dep_zip.namelist(): # Zip entry names can come in any encoding and in practice we find some jars that have # utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases # and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry # name flavors and under all supported pythons. decoded_file_name = to_bytes(qualified_file_name).decode('utf-8') artifacts_by_file_name[decoded_file_name].add(coordinate.artifact_filename) return artifacts_by_file_name
def test_to_bytes(): # type: () -> None assert isinstance(to_bytes(""), bytes) assert isinstance(to_bytes("abc"), bytes) assert isinstance(to_bytes(b"abc"), bytes) assert isinstance(to_bytes(u"abc"), bytes) assert isinstance(to_bytes(b"abc".decode("latin-1"), encoding=u"utf-8"), bytes) for bad_value in (123, None): with pytest.raises(ValueError): to_bytes(bad_value) # type: ignore[type-var]
def test_osx_platform_intel_issue_523(): def bad_interpreter(include_site_extras=True): return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER, include_site_extras=include_site_extras) interpreter = bad_interpreter(include_site_extras=False) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil. for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): for resolved_dist in resolver.resolve([requirement], cache=cache, # We can't use wheels since we're bootstrapping them. precedence=(SourcePackage, EggPackage), interpreter=interpreter): dist = resolved_dist.distribution interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, precedence=(SourcePackage, WheelPackage), interpreter=interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) args = ['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'] env = os.environ.copy() env['PEX_VERBOSE'] = '1' process = pex.run(args=args, env=env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert 0 == process.returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format(process.returncode, stderr) ) # Verify this all worked under the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = '.'.join(release.split('.')[:2]) assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_activate_extras_issue_615(): with yield_pex_builder() as pb: for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter): pb.add_requirement(resolved_dist.requirement) pb.add_dist_location(resolved_dist.distribution.location) pb.set_script('pex') pb.freeze() process = PEX(pb.path(), interpreter=pb.interpreter).run(args=['--version'], env={'PEX_VERBOSE': '9'}, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert 0 == process.returncode, ( 'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr) ) assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()
def build(self, filename, bytecode_compile=True, deterministic_timestamp=False): """Package the PEX into a zipfile. :param filename: The filename where the PEX should be stored. :param bytecode_compile: If True, precompile .py files into .pyc files. :param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps. If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the PEXBuilder immutable. """ if not self._frozen: self.freeze(bytecode_compile=bytecode_compile) tmp_zip = filename + "~" try: os.unlink(tmp_zip) self._logger.warning( "Previous binary unexpectedly exists, cleaning: {}".format( tmp_zip)) except OSError: # The expectation is that the file does not exist, so continue pass with safe_open(tmp_zip, "ab") as pexfile: assert os.path.getsize(pexfile.name) == 0 pexfile.write(to_bytes("{}\n".format(self._shebang))) with TRACER.timed("Zipping PEX file."): self._chroot.zip( tmp_zip, mode="a", deterministic_timestamp=deterministic_timestamp, # When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions # as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since # those installed wheels reside in a shared cache, they can be in-use by other # processes and so their code may be in the process of being bytecode compiled as we # attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary # pyc files that we should avoid copying since they are unuseful and inherently # racy. exclude_file=is_pyc_temporary_file, ) if os.path.exists(filename): os.unlink(filename) os.rename(tmp_zip, filename) chmod_plus_x(filename)
def _get_external_dependencies(self, binary_target): artifacts_by_file_name = defaultdict(set) for basedir, externaljar in self.list_external_jar_dependencies(binary_target): external_dep = os.path.join(basedir, externaljar) self.context.log.debug(' scanning %s' % external_dep) with open_zip64(external_dep) as dep_zip: for qualified_file_name in dep_zip.namelist(): # Zip entry names can come in any encoding and in practice we find some jars that have # utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases # and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry # name flavors and under all supported pythons. decoded_file_name = to_bytes(qualified_file_name).decode('utf-8') if os.path.basename(decoded_file_name).lower() in self._excludes: continue jar_name = os.path.basename(external_dep) if (not self._isdir(decoded_file_name)) and Manifest.PATH != decoded_file_name: artifacts_by_file_name[decoded_file_name].add(jar_name) return artifacts_by_file_name
def test_activate_extras_issue_615(): # type: () -> None with yield_pex_builder() as pb: for resolved_dist in resolver.resolve(["pex[requests]==1.6.3"], interpreter=pb.interpreter): pb.add_requirement(resolved_dist.requirement) pb.add_dist_location(resolved_dist.distribution.location) pb.set_script("pex") pb.freeze() process = PEX(pb.path(), interpreter=pb.interpreter).run( args=["--version"], env={"PEX_VERBOSE": "9"}, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() assert 0 == process.returncode, "Process failed with exit code {} and output:\n{}".format( process.returncode, stderr ) assert to_bytes("{} 1.6.3".format(os.path.basename(pb.path()))) == stdout.strip()
def compile(self, root, relpaths): """Compiles the given python source files using this compiler's interpreter. :param string root: The root path all the source files are found under. :param list relpaths: The relative paths from the `root` of the source files to compile. :returns: A list of relative paths of the compiled bytecode files. :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files. """ with named_temporary_file() as fp: fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8')) fp.flush() try: _, out, _ = self._interpreter.execute(args=[fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( 'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr) ) return out.splitlines()
def compile(self, root, relpaths): """Compiles the given python source files using this compiler's interpreter. :param string root: The root path all the source files are found under. :param list relpaths: The realtive paths from the `root` of the source files to compile. :returns: A list of relative paths of the compiled bytecode files. :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files. """ with named_temporary_file() as fp: fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8')) fp.flush() try: out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( 'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr) ) return out.splitlines()
def _prepare_code(self): self._pex_info.code_hash = CacheHelper.pex_code_hash( self._chroot.path()) self._pex_info.pex_hash = hashlib.sha1( self._pex_info.dump().encode("utf-8")).hexdigest() self._chroot.write(self._pex_info.dump().encode("utf-8"), PexInfo.PATH, label="manifest") bootstrap = BOOTSTRAP_ENVIRONMENT.format( bootstrap_dir=BOOTSTRAP_DIR, pex_root=self._pex_info.raw_pex_root, pex_hash=self._pex_info.pex_hash, interpreter_constraints=self._pex_info.interpreter_constraints, pex_path=self._pex_info.pex_path, is_unzip=self._pex_info.unzip, is_venv=self._pex_info.venv, ) self._chroot.write(to_bytes(self._preamble + "\n" + bootstrap), "__main__.py", label="main")
def _get_external_dependencies(self, binary_target): artifacts_by_file_name = defaultdict(set) for basedir, externaljar in self.list_external_jar_dependencies( binary_target): external_dep = os.path.join(basedir, externaljar) self.context.log.debug(' scanning %s' % external_dep) with closing(ZipFile(external_dep)) as dep_zip: for qualified_file_name in dep_zip.namelist(): # Zip entry names can come in any encoding and in practice we find some jars that have # utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases # and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry # name flavors and under all supported pythons. decoded_file_name = to_bytes(qualified_file_name).decode( 'utf-8') if os.path.basename( decoded_file_name).lower() in self._excludes: continue jar_name = os.path.basename(external_dep) if (not self._isdir(decoded_file_name) ) and Manifest.PATH != decoded_file_name: artifacts_by_file_name[decoded_file_name].add(jar_name) return artifacts_by_file_name
def compile(self, root, relpaths): # type: (str, Iterable[str]) -> List[Text] """Compiles the given python source files using this compiler's interpreter. :param root: The root path all the source files are found under. :param relpaths: The relative paths from the `root` of the source files to compile. :returns: A list of relative paths of the compiled bytecode files. :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files. """ with named_temporary_file() as fp: fp.write( to_bytes(_COMPILER_MAIN % {"root": root, "relpaths": relpaths}, encoding="utf-8") ) fp.flush() try: _, out, _ = self._interpreter.execute(args=[fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( "encountered %r during bytecode compilation.\nstderr was:\n%s\n" % (e, e.stderr) ) # TODO(#1034): remove the ignore once interpreter.py is updated. return out.splitlines() # type: ignore[no-any-return]
def test_pex_sys_exit_prints_objects(): _test_sys_exit('Exception("derp")', to_bytes('derp\n'), 1)
def test_pex_sys_exit_doesnt_print_none(): _test_sys_exit('', to_bytes(''), 0)
def _prepare_main(self): self._chroot.write(to_bytes(self._preamble + '\n' + BOOTSTRAP_ENVIRONMENT), '__main__.py', label='main')
def write_source(path, valid=True): with safe_open(path, 'wb') as fp: fp.write(to_bytes('basename = %r\n' % os.path.basename(path))) if not valid: fp.write(to_bytes('invalid!\n'))
def write_source(path, valid=True): # type: (str, bool) -> None with safe_open(path, "wb") as fp: fp.write(to_bytes("basename = %r\n" % os.path.basename(path))) if not valid: fp.write(to_bytes("invalid!\n"))
def test_osx_platform_intel_issue_523(): def bad_interpreter(): return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored # setuptools and wheel extras. with nested(yield_pex_builder(interpreter=bad_interpreter()), temporary_filename()) as ( pb, pex_file, ): for resolved_dist in resolver.resolve(["psutil==5.4.3"], cache=cache, interpreter=pb.interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) def run(args, **env): pex_env = os.environ.copy() pex_env["PEX_VERBOSE"] = "1" pex_env.update(**env) process = pex.run( args=args, env=pex_env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() return process.returncode, stdout, stderr returncode, _, stderr = run(["-c", "import psutil"]) assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format( returncode, stderr) returncode, stdout, stderr = run(["-c", "import pkg_resources"]) assert 0 != returncode, ( "Isolated pex process succeeded but should not have found pkg-resources:\n" "STDOUT:\n" "{}\n" "STDERR:\n" "{}".format(stdout, stdout, stderr)) returncode, stdout, stderr = run( [ "-c", "import pkg_resources; print(pkg_resources.get_supported_platform())" ], # Let the bad interpreter site-packages setuptools leak in. PEX_INHERIT_PATH="1", ) assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format( returncode, stderr) # Verify this worked along side the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = ".".join(release.split(".")[:2]) assert to_bytes( "macosx-{}-intel".format(major_minor)) == stdout.strip()
def _prepare_main(self): self._chroot.write(to_bytes(self._preamble + "\n" + BOOTSTRAP_ENVIRONMENT), "__main__.py", label="main")
def test_pex_sys_exit_doesnt_print_none(): _test_sys_exit("", to_bytes(""), 0)