def test_clp_preamble_file():
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes('print "foo!"'))
tmpfile.flush()
parser, resolver_options_builder = configure_clp()
options, reqs = parser.parse_args(args=['--preamble-file', tmpfile.name])
assert options.preamble_file == tmpfile.name
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder._preamble == to_bytes('print "foo!"')
def test_pex_sys_exit_prints_non_numeric_value_no_traceback(): text = 'something went wrong' sys_exit_arg = '"' + text + '"' # encode the string somehow that's compatible with 2 and 3 expected_output = to_bytes(text) + b'\n' _test_sys_exit(sys_exit_arg, expected_output, 1)
def test_iter_pth_paths(mock_exists):
# Ensure path checking always returns True for dummy paths.
mock_exists.return_value = True
with temporary_dir() as tmpdir:
in_tmp = lambda f: os.path.join(tmpdir, f)
PTH_TEST_MAPPING = {
# A mapping of .pth file content -> expected paths.
'/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python\n': [
'/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python'
],
'relative_path\nrelative_path2\n\nrelative_path3': [
in_tmp('relative_path'),
in_tmp('relative_path2'),
in_tmp('relative_path3')
],
'duplicate_path\nduplicate_path': [in_tmp('duplicate_path')],
'randompath\nimport nosuchmodule\n': [in_tmp('randompath')],
'import nosuchmodule\nfoo': [],
'import nosuchmodule\n': [],
'import bad)syntax\n': [],
}
for i, pth_content in enumerate(PTH_TEST_MAPPING):
pth_tmp_path = os.path.abspath(os.path.join(tmpdir, 'test%s.pth' % i))
with open(pth_tmp_path, 'wb') as f:
f.write(to_bytes(pth_content))
assert sorted(PTH_TEST_MAPPING[pth_content]) == sorted(list(iter_pth_paths(pth_tmp_path)))
def build(self, filename, bytecode_compile=True, deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
try:
os.unlink(filename + '~')
self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
if os.path.dirname(filename):
safe_mkdir(os.path.dirname(filename))
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes('%s\n' % self._shebang))
self._chroot.zip(filename + '~', mode='a', deterministic_timestamp=deterministic_timestamp)
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def convert(input):
if isinstance(input, dict):
out = dict()
for key, value in input.items():
out[convert(key)] = convert(value)
return out
elif isinstance(input, list):
return [convert(element) for element in input]
elif isinstance(input, string):
return to_bytes(input)
else:
return input
def test_to_bytes():
assert isinstance(to_bytes(''), bytes)
assert isinstance(to_bytes('abc'), bytes)
assert isinstance(to_bytes(b'abc'), bytes)
assert isinstance(to_bytes(u'abc'), bytes)
assert isinstance(to_bytes(b'abc'.decode('latin-1'), encoding='utf-8'), bytes)
for bad_values in (123, None):
with pytest.raises(ValueError):
to_bytes(bad_values)
def test_clp_preamble_file():
# type: () -> None
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes('print "foo!"'))
tmpfile.flush()
parser = configure_clp()
options = parser.parse_args(args=["--preamble-file", tmpfile.name])
assert options.preamble_file == tmpfile.name
pex_builder = build_pex(options.requirements, options)
assert pex_builder._preamble == 'print "foo!"'
def test_clp_preamble_file():
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes('print "foo!"'))
tmpfile.flush()
parser = configure_clp()
options, reqs = parser.parse_args(
args=['--preamble-file', tmpfile.name])
assert options.preamble_file == tmpfile.name
pex_builder = build_pex(reqs, options)
assert pex_builder._preamble == 'print "foo!"'
def test_to_bytes():
assert isinstance(to_bytes(''), bytes)
assert isinstance(to_bytes('abc'), bytes)
assert isinstance(to_bytes(b'abc'), bytes)
assert isinstance(to_bytes(u'abc'), bytes)
assert isinstance(to_bytes(b'abc'.decode('latin-1'), encoding='utf-8'), bytes)
for bad_value in (123, None):
with pytest.raises(ValueError):
to_bytes(bad_value)
def convert(input):
if isinstance(input, dict):
out = dict()
for key, value in input.items():
out[convert(key)] = convert(value)
return out
elif isinstance(input, list):
return [convert(element) for element in input]
elif isinstance(input, string):
return to_bytes(input)
else:
return input
def test_get_script_from_whl():
whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl'
dists = list(find_wheels_in_zip(zipimport.zipimporter(whl_path), whl_path))
assert len(dists) == 1
dist = dists[0]
assert 'aws-cfn-bootstrap' == dist.project_name
script_path, script_content = get_script_from_whl('cfn-signal', dist)
assert os.path.join(whl_path, 'aws_cfn_bootstrap-1.4.data/scripts/cfn-signal') == script_path
assert script_content.startswith(to_bytes('#!')), 'Expected a `scripts`-style script w/shebang.'
assert (None, None) == get_script_from_whl('non_existent_script', dist)
def test_to_bytes():
assert isinstance(to_bytes(""), bytes)
assert isinstance(to_bytes("abc"), bytes)
assert isinstance(to_bytes(b"abc"), bytes)
assert isinstance(to_bytes(u"abc"), bytes)
assert isinstance(to_bytes(b"abc".decode("latin-1"), encoding="utf-8"),
bytes)
for bad_value in (123, None):
with pytest.raises(ValueError):
to_bytes(bad_value)
def test_get_script_from_whl():
whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl'
dists = list(find_wheels_in_zip(zipimport.zipimporter(whl_path), whl_path))
assert len(dists) == 1
dist = dists[0]
assert 'aws-cfn-bootstrap' == dist.project_name
script_path, script_content = get_script_from_whl('cfn-signal', dist)
assert os.path.join(whl_path, 'aws_cfn_bootstrap-1.4.data/scripts/cfn-signal') == script_path
assert script_content.startswith(to_bytes('#!')), 'Expected a `scripts`-style script w/shebang.'
assert (None, None) == get_script_from_whl('non_existent_script', dist)
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for external_dep, coordinate in self.list_external_jar_dependencies(binary_target):
self.context.log.debug(' scanning {} from {}'.format(coordinate, external_dep))
with open_zip(external_dep) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode('utf-8')
artifacts_by_file_name[decoded_file_name].add(coordinate.artifact_filename)
return artifacts_by_file_name
def test_to_bytes():
# type: () -> None
assert isinstance(to_bytes(""), bytes)
assert isinstance(to_bytes("abc"), bytes)
assert isinstance(to_bytes(b"abc"), bytes)
assert isinstance(to_bytes(u"abc"), bytes)
assert isinstance(to_bytes(b"abc".decode("latin-1"), encoding=u"utf-8"),
bytes)
for bad_value in (123, None):
with pytest.raises(ValueError):
to_bytes(bad_value) # type: ignore[type-var]
def test_osx_platform_intel_issue_523():
def bad_interpreter(include_site_extras=True):
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER,
include_site_extras=include_site_extras)
interpreter = bad_interpreter(include_site_extras=False)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for resolved_dist in resolver.resolve([requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)
with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
args = ['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())']
env = os.environ.copy()
env['PEX_VERBOSE'] = '1'
process = pex.run(args=args,
env=env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(process.returncode, stderr)
)
# Verify this all worked under the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def test_activate_extras_issue_615():
with yield_pex_builder() as pb:
for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter):
pb.add_requirement(resolved_dist.requirement)
pb.add_dist_location(resolved_dist.distribution.location)
pb.set_script('pex')
pb.freeze()
process = PEX(pb.path(), interpreter=pb.interpreter).run(args=['--version'],
env={'PEX_VERBOSE': '9'},
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr)
)
assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()
def build(self,
filename,
bytecode_compile=True,
deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
tmp_zip = filename + "~"
try:
os.unlink(tmp_zip)
self._logger.warning(
"Previous binary unexpectedly exists, cleaning: {}".format(
tmp_zip))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with safe_open(tmp_zip, "ab") as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes("{}\n".format(self._shebang)))
with TRACER.timed("Zipping PEX file."):
self._chroot.zip(
tmp_zip,
mode="a",
deterministic_timestamp=deterministic_timestamp,
# When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions
# as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since
# those installed wheels reside in a shared cache, they can be in-use by other
# processes and so their code may be in the process of being bytecode compiled as we
# attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary
# pyc files that we should avoid copying since they are unuseful and inherently
# racy.
exclude_file=is_pyc_temporary_file,
)
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmp_zip, filename)
chmod_plus_x(filename)
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for basedir, externaljar in self.list_external_jar_dependencies(binary_target):
external_dep = os.path.join(basedir, externaljar)
self.context.log.debug(' scanning %s' % external_dep)
with open_zip64(external_dep) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode('utf-8')
if os.path.basename(decoded_file_name).lower() in self._excludes:
continue
jar_name = os.path.basename(external_dep)
if (not self._isdir(decoded_file_name)) and Manifest.PATH != decoded_file_name:
artifacts_by_file_name[decoded_file_name].add(jar_name)
return artifacts_by_file_name
def test_activate_extras_issue_615():
# type: () -> None
with yield_pex_builder() as pb:
for resolved_dist in resolver.resolve(["pex[requests]==1.6.3"], interpreter=pb.interpreter):
pb.add_requirement(resolved_dist.requirement)
pb.add_dist_location(resolved_dist.distribution.location)
pb.set_script("pex")
pb.freeze()
process = PEX(pb.path(), interpreter=pb.interpreter).run(
args=["--version"],
env={"PEX_VERBOSE": "9"},
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
assert 0 == process.returncode, "Process failed with exit code {} and output:\n{}".format(
process.returncode, stderr
)
assert to_bytes("{} 1.6.3".format(os.path.basename(pb.path()))) == stdout.strip()
def compile(self, root, relpaths):
"""Compiles the given python source files using this compiler's interpreter.
:param string root: The root path all the source files are found under.
:param list relpaths: The relative paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8'))
fp.flush()
try:
_, out, _ = self._interpreter.execute(args=[fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr)
)
return out.splitlines()
def compile(self, root, relpaths):
"""Compiles the given python source files using this compiler's interpreter.
:param string root: The root path all the source files are found under.
:param list relpaths: The realtive paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8'))
fp.flush()
try:
out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr)
)
return out.splitlines()
def _prepare_code(self):
self._pex_info.code_hash = CacheHelper.pex_code_hash(
self._chroot.path())
self._pex_info.pex_hash = hashlib.sha1(
self._pex_info.dump().encode("utf-8")).hexdigest()
self._chroot.write(self._pex_info.dump().encode("utf-8"),
PexInfo.PATH,
label="manifest")
bootstrap = BOOTSTRAP_ENVIRONMENT.format(
bootstrap_dir=BOOTSTRAP_DIR,
pex_root=self._pex_info.raw_pex_root,
pex_hash=self._pex_info.pex_hash,
interpreter_constraints=self._pex_info.interpreter_constraints,
pex_path=self._pex_info.pex_path,
is_unzip=self._pex_info.unzip,
is_venv=self._pex_info.venv,
)
self._chroot.write(to_bytes(self._preamble + "\n" + bootstrap),
"__main__.py",
label="main")
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for basedir, externaljar in self.list_external_jar_dependencies(
binary_target):
external_dep = os.path.join(basedir, externaljar)
self.context.log.debug(' scanning %s' % external_dep)
with closing(ZipFile(external_dep)) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode(
'utf-8')
if os.path.basename(
decoded_file_name).lower() in self._excludes:
continue
jar_name = os.path.basename(external_dep)
if (not self._isdir(decoded_file_name)
) and Manifest.PATH != decoded_file_name:
artifacts_by_file_name[decoded_file_name].add(jar_name)
return artifacts_by_file_name
def compile(self, root, relpaths):
# type: (str, Iterable[str]) -> List[Text]
"""Compiles the given python source files using this compiler's interpreter.
:param root: The root path all the source files are found under.
:param relpaths: The relative paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(
to_bytes(_COMPILER_MAIN % {"root": root, "relpaths": relpaths}, encoding="utf-8")
)
fp.flush()
try:
_, out, _ = self._interpreter.execute(args=[fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
"encountered %r during bytecode compilation.\nstderr was:\n%s\n" % (e, e.stderr)
)
# TODO(#1034): remove the ignore once interpreter.py is updated.
return out.splitlines() # type: ignore[no-any-return]
def test_pex_sys_exit_prints_objects():
_test_sys_exit('Exception("derp")', to_bytes('derp\n'), 1)
def test_pex_sys_exit_doesnt_print_none():
_test_sys_exit('', to_bytes(''), 0)
def _prepare_main(self):
self._chroot.write(to_bytes(self._preamble + '\n' + BOOTSTRAP_ENVIRONMENT),
'__main__.py', label='main')
def write_source(path, valid=True):
with safe_open(path, 'wb') as fp:
fp.write(to_bytes('basename = %r\n' % os.path.basename(path)))
if not valid:
fp.write(to_bytes('invalid!\n'))
def write_source(path, valid=True):
# type: (str, bool) -> None
with safe_open(path, "wb") as fp:
fp.write(to_bytes("basename = %r\n" % os.path.basename(path)))
if not valid:
fp.write(to_bytes("invalid!\n"))
def write_source(path, valid=True):
with safe_open(path, 'wb') as fp:
fp.write(to_bytes('basename = %r\n' % os.path.basename(path)))
if not valid:
fp.write(to_bytes('invalid!\n'))
def test_osx_platform_intel_issue_523():
def bad_interpreter():
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
# setuptools and wheel extras.
with nested(yield_pex_builder(interpreter=bad_interpreter()),
temporary_filename()) as (
pb,
pex_file,
):
for resolved_dist in resolver.resolve(["psutil==5.4.3"],
cache=cache,
interpreter=pb.interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
def run(args, **env):
pex_env = os.environ.copy()
pex_env["PEX_VERBOSE"] = "1"
pex_env.update(**env)
process = pex.run(
args=args,
env=pex_env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
return process.returncode, stdout, stderr
returncode, _, stderr = run(["-c", "import psutil"])
assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format(
returncode, stderr)
returncode, stdout, stderr = run(["-c", "import pkg_resources"])
assert 0 != returncode, (
"Isolated pex process succeeded but should not have found pkg-resources:\n"
"STDOUT:\n"
"{}\n"
"STDERR:\n"
"{}".format(stdout, stdout, stderr))
returncode, stdout, stderr = run(
[
"-c",
"import pkg_resources; print(pkg_resources.get_supported_platform())"
],
# Let the bad interpreter site-packages setuptools leak in.
PEX_INHERIT_PATH="1",
)
assert 0 == returncode, "Process failed with exit code {} and stderr:\n{}".format(
returncode, stderr)
# Verify this worked along side the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = ".".join(release.split(".")[:2])
assert to_bytes(
"macosx-{}-intel".format(major_minor)) == stdout.strip()
def _prepare_main(self):
self._chroot.write(to_bytes(self._preamble + "\n" +
BOOTSTRAP_ENVIRONMENT),
"__main__.py",
label="main")
def test_pex_sys_exit_doesnt_print_none():
_test_sys_exit("", to_bytes(""), 0)
def _prepare_main(self):
self._chroot.write(to_bytes(self._preamble + '\n' + BOOTSTRAP_ENVIRONMENT),
'__main__.py', label='main')
def test_pex_sys_exit_prints_objects():
_test_sys_exit('Exception("derp")', to_bytes('derp\n'), 1)
def test_pex_sys_exit_doesnt_print_none():
_test_sys_exit('', to_bytes(''), 0)
