def test_pex_get_int():
# type: () -> None
with pytest.raises(NoValueError):
Variables()._get_int("HELLO")
assert Variables(environ={"HELLO": "23"})._get_int("HELLO") == 23
with pytest.raises(SystemExit):
assert Variables(environ={"HELLO": "welp"})._get_int("HELLO")
def test_pex_string_variables():
Variables(environ={})._get_string('NOT_HERE') is None
Variables(environ={})._get_string('NOT_HERE', default='lolol') == 'lolol'
Variables(environ={'HERE': 'stuff'})._get_string('HERE') == 'stuff'
Variables(environ={
'HERE': 'stuff'
})._get_string('HERE', default='lolol') == 'stuff'
def test_pex_get_int():
assert Variables()._get_int("HELLO") is None
assert Variables()._get_int("HELLO", default=42) == 42
assert Variables(environ={"HELLO": 23})._get_int("HELLO") == 23
assert Variables(environ={"HELLO": 23})._get_int("HELLO", default=42) == 23
with pytest.raises(SystemExit):
assert Variables(environ={"HELLO": "welp"})._get_int("HELLO")
def test_pex_get_int():
assert Variables()._get_int('HELLO') is None
assert Variables()._get_int('HELLO', default=42) == 42
assert Variables(environ={'HELLO': 23})._get_int('HELLO') == 23
assert Variables(environ={'HELLO': 23})._get_int('HELLO', default=42) == 23
with pytest.raises(SystemExit):
assert Variables(environ={'HELLO': 'welp'})._get_int('HELLO')
def test_pex_string_variables():
# type: () -> None
assert Variables(environ={})._get_string("NOT_HERE") is None
assert Variables(environ={})._get_string("NOT_HERE",
default="lolol") == "lolol"
assert Variables(environ={"HERE": "stuff"})._get_string("HERE") == "stuff"
assert Variables(environ={
"HERE": "stuff"
})._get_string("HERE", default="lolol") == "stuff"
def test_from_env():
# type: () -> None
with temporary_dir() as td:
pex_root = os.path.realpath(os.path.join(td, "pex_root"))
environ = dict(
PEX_ROOT=pex_root,
PEX_MODULE="entry:point",
PEX_SCRIPT="script.sh",
PEX_FORCE_LOCAL="true",
PEX_UNZIP="true",
PEX_INHERIT_PATH="prefer",
PEX_IGNORE_ERRORS="true",
PEX_ALWAYS_CACHE="true",
)
info = dict(
pex_root=pex_root,
entry_point="entry:point",
script="script.sh",
zip_safe=False,
unzip=True,
inherit_path=True,
ignore_errors=True,
always_write_cache=True,
)
assert_same_info(PexInfo(info=info),
PexInfo.from_env(env=Variables(environ=environ)))
def exercise_warnings(pex_info, **env):
# type: (PexInfo, **str) -> List[warnings.WarningMessage]
with warnings.catch_warnings(record=True) as events:
pex_warnings.configure_warnings(pex_info, env=Variables(environ=env))
pex_warnings.warn("test")
assert events is not None
return events
def test_pexrc_precedence():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=FORTYTWO")
pexrc.flush()
v = Variables(rc=pexrc.name, environ={"HELLO": "42"})
assert v._get_int("HELLO") == 42
def test_pex_vars_set():
v = Variables(environ={})
v.set('HELLO', '42')
assert v._get_int('HELLO') == 42
v.delete('HELLO')
assert v._get_int('HELLO') is None
assert {} == v.copy()
def test_rc_ignore():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=FORTYTWO")
pexrc.flush()
v = Variables(rc=pexrc.name, environ={"PEX_IGNORE_RCFILES": "True"})
assert "HELLO" not in v._environ
def test_pex_from_rc():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=42")
pexrc.flush()
v = Variables(rc=pexrc.name)
assert v._get_int("HELLO") == 42
def test_pex_get_kv():
# type: () -> None
v = Variables(environ={})
assert v._get_kv("HELLO") is None
assert v._get_kv("=42") is None
assert v._get_kv("TOO=MANY=COOKS") is None
assert v._get_kv("THIS=WORKS") == ["THIS", "WORKS"]
def assert_pex_vars_hermetic():
v = Variables()
assert os.environ == v.copy()
existing = os.environ.get('TEST')
expected = (existing or '') + 'different'
assert expected != existing
with environment_as(TEST=expected):
assert expected != v.copy().get('TEST')
def assert_pex_vars_hermetic():
# type: () -> None
v = Variables()
assert os.environ.copy() == v.copy()
existing = os.environ.get("TEST")
expected = (existing or "") + "different"
assert expected != existing
with environment_as(TEST=expected):
assert expected != v.copy().get("TEST")
def test_requests_context_retries_connect_timeout_retries_exhausted():
with mock.patch.object(
requests.packages.urllib3.connectionpool.HTTPConnectionPool,
'_make_request') as mock_make_request:
url, mock_make_request.side_effect = timeout_side_effect(num_timeouts=3)
env = Variables(environ={'PEX_HTTP_RETRIES': '2'})
context = RequestsContext(verify=False, env=env)
with pytest.raises(Context.Error):
context.read(Link.wrap(url))
def _scrub_import_environment(sys_modules_whitelist: typing.List[str],
logger: typing.Callable):
"""Scrubs sys.path and sys.modules to a raw state.
WARNING: This will irreversably mutate sys.path and sys.modules each time it's called.
"""
pex_root = pathlib.Path(Variables().PEX_ROOT)
# A generator that emits sys.path elements
def scrubbed_sys_path():
"""Yields a scrubbed version of sys.path."""
for p in sys.path[:]:
if not isinstance(p, str):
yield p
# Scrub any/all pex locations from sys.path.
pp = pathlib.Path(p)
if pex_root not in pp.parents:
yield p
def scrub_from_sys_modules():
"""Yields keys of sys.modules as candidates for scrubbing/removal."""
for k, m in sys.modules.items():
if k in sys_modules_whitelist:
continue
if hasattr(m, '__file__') and m.__file__ is not None:
mp = pathlib.Path(m.__file__)
if pex_root in mp.parents:
yield k
def scrub_env():
# Replace sys.path with a scrubbed version.
sys.path[:] = list(scrubbed_sys_path())
# Drop module cache references from sys.modules.
modules_to_scrub = list(scrub_from_sys_modules())
for m in modules_to_scrub:
del sys.modules[m]
logger(
'Scrubbing sys.path and sys.modules in preparation for pex bootstrap\n'
)
logger(f'sys.path contains {len(sys.path)} items, '
f'sys.modules contains {len(sys.modules)} keys\n')
# Scrub environment.
scrub_env()
logger(f'sys.path now contains {len(sys.path)} items, '
f'sys.modules now contains {len(sys.modules)} keys\n')
def test_pex_vars_value_or(tmpdir):
# type: (Any) -> None
v = Variables(environ={})
assert v.PEX_ROOT is not None, "Expected PEX_ROOT to be a defaulted variable."
pex_root = str(tmpdir)
assert pex_root == Variables.PEX_ROOT.value_or(v, pex_root)
unwriteable_pex_root = os.path.join(pex_root, "unwriteable")
os.mkdir(unwriteable_pex_root, 0o444)
assert unwriteable_pex_root != Variables.PEX_ROOT.value_or(
v, unwriteable_pex_root
), ("Expected the fallback to be validated, and in the case of PEX_ROOT, replaced with a "
"writeable tmp dir")
def test_pex_vars_defaults_stripped():
# type: () -> None
v = Variables(environ={})
# bool
assert v.PEX_ALWAYS_CACHE is not None
assert Variables.PEX_ALWAYS_CACHE.strip_default(v) is None
# string
assert v.PEX_PATH is not None
assert Variables.PEX_PATH.strip_default(v) is None
# int
assert v.PEX_VERBOSE is not None
assert Variables.PEX_VERBOSE.strip_default(v) is None
def test_pex_vars_defaults_stripped():
v = Variables(environ={})
stripped = v.strip_defaults()
# bool
assert v.PEX_ALWAYS_CACHE is not None
assert stripped.PEX_ALWAYS_CACHE is None
# string
assert v.PEX_PATH is not None
assert stripped.PEX_PATH is None
# int
assert v.PEX_VERBOSE is not None
assert stripped.PEX_VERBOSE is None
def test_pex_bool_variables():
# type: () -> None
assert Variables(environ={})._maybe_get_bool("NOT_HERE") is None
with pytest.raises(NoValueError):
Variables(environ={})._get_bool("NOT_HERE")
for value in ("0", "faLsE", "false"):
assert Variables(environ={"HERE": value})._get_bool("HERE") is False
for value in ("1", "TrUe", "true"):
assert Variables(environ={"HERE": value})._get_bool("HERE") is True
with pytest.raises(SystemExit):
Variables(environ={"HERE": "garbage"})._get_bool("HERE")
# end to end
assert Variables().PEX_ALWAYS_CACHE is False
assert Variables({"PEX_ALWAYS_CACHE": "1"}).PEX_ALWAYS_CACHE is True
def test_pex_bool_variables():
Variables(environ={})._get_bool("NOT_HERE", default=False) is False
Variables(environ={})._get_bool("NOT_HERE", default=True) is True
for value in ("0", "faLsE", "false"):
for default in (True, False):
Variables(environ={"HERE": value})._get_bool("HERE", default=default) is False
for value in ("1", "TrUe", "true"):
for default in (True, False):
Variables(environ={"HERE": value})._get_bool("HERE", default=default) is True
with pytest.raises(SystemExit):
Variables(environ={"HERE": "garbage"})._get_bool("HERE")
# end to end
assert Variables().PEX_ALWAYS_CACHE is False
assert Variables({"PEX_ALWAYS_CACHE": "1"}).PEX_ALWAYS_CACHE is True
def test_pex_bool_variables():
Variables(environ={})._get_bool('NOT_HERE', default=False) is False
Variables(environ={})._get_bool('NOT_HERE', default=True) is True
for value in ('0', 'faLsE', 'false'):
for default in (True, False):
Variables(environ={'HERE': value})._get_bool('HERE', default=default) is False
for value in ('1', 'TrUe', 'true'):
for default in (True, False):
Variables(environ={'HERE': value})._get_bool('HERE', default=default) is True
with pytest.raises(SystemExit):
Variables(environ={'HERE': 'garbage'})._get_bool('HERE')
# end to end
assert Variables().PEX_ALWAYS_CACHE is False
assert Variables({'PEX_ALWAYS_CACHE': '1'}).PEX_ALWAYS_CACHE is True
def test_pex_root_unwriteable():
with temporary_dir() as td:
pex_root = os.path.realpath(os.path.join(td, "pex_root"))
os.mkdir(pex_root, 0o444)
env = Variables(environ=dict(PEX_ROOT=pex_root))
with warnings.catch_warnings(record=True) as log:
assert pex_root != env.PEX_ROOT
assert 1 == len(log)
message = log[0].message
assert isinstance(message, PEXWarning)
assert pex_root in str(message)
assert env.PEX_ROOT in str(message)
assert env.PEX_ROOT == env.PEX_ROOT, (
"When an ephemeral PEX_ROOT is materialized it should be stable.")
def test_from_env():
pex_root = os.path.realpath('/pex_root')
environ = dict(PEX_ROOT=pex_root,
PEX_MODULE='entry:point',
PEX_SCRIPT='script.sh',
PEX_FORCE_LOCAL='true',
PEX_INHERIT_PATH='true',
PEX_IGNORE_ERRORS='true',
PEX_ALWAYS_CACHE='true')
info = dict(pex_root=pex_root,
entry_point='entry:point',
script='script.sh',
zip_safe=False,
inherit_path=True,
ignore_errors=True,
always_write_cache=True)
assert_same_info(PexInfo(info=info), PexInfo.from_env(env=Variables(environ=environ)))
def exercise_warnings(pex_info, **env):
with warnings.catch_warnings(record=True) as events:
pex_warnings.configure_warnings(pex_info, env=Variables(environ=env))
pex_warnings.warn('test')
return events
def build_pex(reqs, options, cache=None):
interpreters = None # Default to the current interpreter.
pex_python_path = options.python_path # If None, this will result in using $PATH.
# TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag.
if pex_python_path is None and (options.rc_file
or not ENV.PEX_IGNORE_RCFILES):
rc_variables = Variables(rc=options.rc_file)
pex_python_path = rc_variables.PEX_PYTHON_PATH
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed("Resolving interpreters", V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" %
full_path_or_basename)
return interp
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed("Resolving interpreters", V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
try:
interpreters = list(
iter_compatible_interpreters(
path=pex_python_path,
interpreter_constraints=constraints))
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Could not find a compatible interpreter."),
CANNOT_SETUP_INTERPRETER,
)
platforms = OrderedSet(options.platforms)
interpreters = interpreters or []
if options.platforms and options.resolve_local_platforms:
with TRACER.timed(
"Searching for local interpreters matching {}".format(
", ".join(map(str, platforms)))):
candidate_interpreters = OrderedSet(
iter_compatible_interpreters(path=pex_python_path))
candidate_interpreters.add(PythonInterpreter.get())
for candidate_interpreter in candidate_interpreters:
resolved_platforms = candidate_interpreter.supported_platforms.intersection(
platforms)
if resolved_platforms:
for resolved_platform in resolved_platforms:
TRACER.log("Resolved {} for platform {}".format(
candidate_interpreter, resolved_platform))
platforms.remove(resolved_platform)
interpreters.append(candidate_interpreter)
if platforms:
TRACER.log(
"Could not resolve a local interpreter for {}, will resolve only binary distributions "
"for {}.".format(
", ".join(map(str, platforms)),
"this platform"
if len(platforms) == 1 else "these platforms",
))
interpreter = (PythonInterpreter.latest_release_of_min_compatible_version(
interpreters) if interpreters else None)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
include_tools=options.include_tools or options.venv,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead.")
for directory in OrderedSet(options.sources_directory +
options.resources_directory):
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.unzip = options.unzip
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv
pex_info.venv_copies = options.venv_copies
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = InheritPath.for_value(options.inherit_path)
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
indexes = compute_indexes(options)
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(reqs +
options.requirement_files)):
if options.cache_ttl:
pex_warnings.warn(
"The --cache-ttl option is deprecated and no longer has any effect."
)
if options.headers:
pex_warnings.warn(
"The --header option is deprecated and no longer has any effect."
)
network_configuration = NetworkConfiguration(
retries=options.retries,
timeout=options.timeout,
proxy=options.proxy,
cert=options.cert,
client_cert=options.client_cert,
)
try:
if options.pex_repository:
with TRACER.timed("Resolving requirements from PEX {}.".format(
options.pex_repository)):
resolveds = resolve_from_pex(
pex=options.pex_repository,
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
network_configuration=network_configuration,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
manylinux=options.manylinux,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
indexes=indexes,
find_links=options.find_links,
resolver_version=ResolverVersion.for_value(
options.resolver_version),
network_configuration=network_configuration,
cache=cache,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors,
)
for resolved_dist in resolveds:
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(
resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(str(e))
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def test_pex_get_kv():
v = Variables(environ={})
assert v._get_kv('HELLO') is None
assert v._get_kv('=42') is None
assert v._get_kv('TOO=MANY=COOKS') is None
assert v._get_kv('THIS=WORKS') == ['THIS', 'WORKS']
def test_pex_from_rc():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=42')
pexrc.flush()
v = Variables(rc=pexrc.name)
assert v._get_int('HELLO') == 42
def test_pexrc_precedence():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'HELLO': 42})
assert v._get_int('HELLO') == 42
def test_rc_ignore():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'PEX_IGNORE_RCFILES': 'True'})
assert 'HELLO' not in v._environ