def test_pex_from_rc(): # type: () -> None with named_temporary_file(mode="w") as pexrc: pexrc.write("HELLO=42") pexrc.flush() v = Variables(rc=pexrc.name) assert v._get_int("HELLO") == 42
def test_pex_get_int(): # type: () -> None with pytest.raises(NoValueError): Variables()._get_int("HELLO") assert Variables(environ={"HELLO": "23"})._get_int("HELLO") == 23 with pytest.raises(SystemExit): assert Variables(environ={"HELLO": "welp"})._get_int("HELLO")
def test_pexrc_precedence(): # type: () -> None with named_temporary_file(mode="w") as pexrc: pexrc.write("HELLO=FORTYTWO") pexrc.flush() v = Variables(rc=pexrc.name, environ={"HELLO": "42"}) assert v._get_int("HELLO") == 42
def test_pex_string_variables(): Variables(environ={})._get_string('NOT_HERE') is None Variables(environ={})._get_string('NOT_HERE', default='lolol') == 'lolol' Variables(environ={'HERE': 'stuff'})._get_string('HERE') == 'stuff' Variables(environ={ 'HERE': 'stuff' })._get_string('HERE', default='lolol') == 'stuff'
def test_pex_get_int(): assert Variables()._get_int('HELLO') is None assert Variables()._get_int('HELLO', default=42) == 42 assert Variables(environ={'HELLO': 23})._get_int('HELLO') == 23 assert Variables(environ={'HELLO': 23})._get_int('HELLO', default=42) == 23 with pytest.raises(SystemExit): assert Variables(environ={'HELLO': 'welp'})._get_int('HELLO')
def test_pex_get_int(): assert Variables()._get_int("HELLO") is None assert Variables()._get_int("HELLO", default=42) == 42 assert Variables(environ={"HELLO": 23})._get_int("HELLO") == 23 assert Variables(environ={"HELLO": 23})._get_int("HELLO", default=42) == 23 with pytest.raises(SystemExit): assert Variables(environ={"HELLO": "welp"})._get_int("HELLO")
def test_pex_string_variables(): # type: () -> None assert Variables(environ={})._get_string("NOT_HERE") is None assert Variables(environ={})._get_string("NOT_HERE", default="lolol") == "lolol" assert Variables(environ={"HERE": "stuff"})._get_string("HERE") == "stuff" assert Variables(environ={ "HERE": "stuff" })._get_string("HERE", default="lolol") == "stuff"
def assert_pex_vars_hermetic(): v = Variables() assert os.environ == v.copy() existing = os.environ.get('TEST') expected = (existing or '') + 'different' assert expected != existing with environment_as(TEST=expected): assert expected != v.copy().get('TEST')
def test_pex_vars_set(): v = Variables(environ={}) v.set('HELLO', '42') assert v._get_int('HELLO') == 42 v.delete('HELLO') assert v._get_int('HELLO') is None assert {} == v.copy()
def assert_pex_vars_hermetic(): # type: () -> None v = Variables() assert os.environ.copy() == v.copy() existing = os.environ.get("TEST") expected = (existing or "") + "different" assert expected != existing with environment_as(TEST=expected): assert expected != v.copy().get("TEST")
def test_pex_vars_defaults_stripped(): v = Variables(environ={}) stripped = v.strip_defaults() # bool assert v.PEX_ALWAYS_CACHE is not None assert stripped.PEX_ALWAYS_CACHE is None # string assert v.PEX_PATH is not None assert stripped.PEX_PATH is None # int assert v.PEX_VERBOSE is not None assert stripped.PEX_VERBOSE is None
def test_process_pydoc(): def thing(): # no pydoc pass assert Variables.process_pydoc(thing.__doc__) == ('Unknown', 'Unknown') def other_thing(): """Type Properly formatted text. """ assert Variables.process_pydoc(other_thing.__doc__) == ( 'Type', 'Properly formatted text.')
def exercise_warnings(pex_info, **env): # type: (PexInfo, **str) -> List[warnings.WarningMessage] with warnings.catch_warnings(record=True) as events: pex_warnings.configure_warnings(pex_info, env=Variables(environ=env)) pex_warnings.warn("test") assert events is not None return events
def test_process_pydoc(): def thing(): # no pydoc pass assert Variables.process_pydoc(thing.__doc__) == ("Unknown", "Unknown") def other_thing(): """Type Properly formatted text. """ assert Variables.process_pydoc(other_thing.__doc__) == ("Type", "Properly formatted text.")
def print_variable_help(option, option_str, option_value, parser): for variable_name, variable_type, variable_help in Variables.iter_help(): print('\n%s: %s\n' % (variable_name, variable_type)) for line in TextWrapper(initial_indent=' ' * 4, subsequent_indent=' ' * 4).wrap(variable_help): print(line) sys.exit(0)
def test_from_env(): # type: () -> None with temporary_dir() as td: pex_root = os.path.realpath(os.path.join(td, "pex_root")) environ = dict( PEX_ROOT=pex_root, PEX_MODULE="entry:point", PEX_SCRIPT="script.sh", PEX_FORCE_LOCAL="true", PEX_UNZIP="true", PEX_INHERIT_PATH="prefer", PEX_IGNORE_ERRORS="true", PEX_ALWAYS_CACHE="true", ) info = dict( pex_root=pex_root, entry_point="entry:point", script="script.sh", zip_safe=False, unzip=True, inherit_path=True, ignore_errors=True, always_write_cache=True, ) assert_same_info(PexInfo(info=info), PexInfo.from_env(env=Variables(environ=environ)))
def test_rc_ignore(): # type: () -> None with named_temporary_file(mode="w") as pexrc: pexrc.write("HELLO=FORTYTWO") pexrc.flush() v = Variables(rc=pexrc.name, environ={"PEX_IGNORE_RCFILES": "True"}) assert "HELLO" not in v._environ
def __call__(self, parser, namespace, values, option_str=None): for variable_name, variable_type, variable_help in Variables.iter_help(): print("\n%s: %s\n" % (variable_name, variable_type)) for line in TextWrapper(initial_indent=" " * 4, subsequent_indent=" " * 4).wrap( variable_help ): print(line) sys.exit(0)
def get_pex_python_paths(): """Returns a list of paths to Python interpreters as defined in a pexrc file. These are provided by a PEX_PYTHON_PATH in either of '/etc/pexrc', '~/.pexrc'. PEX_PYTHON_PATH defines a colon-separated list of paths to interpreters that a pex can be built and run against. """ ppp = Variables.from_rc().get("PEX_PYTHON_PATH") if ppp: return ppp.split(os.pathsep) else: return []
def test_requests_context_retries_connect_timeout_retries_exhausted(): with mock.patch.object( requests.packages.urllib3.connectionpool.HTTPConnectionPool, '_make_request') as mock_make_request: url, mock_make_request.side_effect = timeout_side_effect(num_timeouts=3) env = Variables(environ={'PEX_HTTP_RETRIES': '2'}) context = RequestsContext(verify=False, env=env) with pytest.raises(Context.Error): context.read(Link.wrap(url))
def get_pex_python_paths(): """Returns a list of paths to Python interpreters as defined in a pexrc file. These are provided by a PEX_PYTHON_PATH in either of '/etc/pexrc', '~/.pexrc'. PEX_PYTHON_PATH defines a colon-separated list of paths to interpreters that a pex can be built and run against. """ ppp = Variables.from_rc().get('PEX_PYTHON_PATH') if ppp: return ppp.split(os.pathsep) else: return []
def test_pex_get_kv(): # type: () -> None v = Variables(environ={}) assert v._get_kv("HELLO") is None assert v._get_kv("=42") is None assert v._get_kv("TOO=MANY=COOKS") is None assert v._get_kv("THIS=WORKS") == ["THIS", "WORKS"]
def _scrub_import_environment(sys_modules_whitelist: typing.List[str], logger: typing.Callable): """Scrubs sys.path and sys.modules to a raw state. WARNING: This will irreversably mutate sys.path and sys.modules each time it's called. """ pex_root = pathlib.Path(Variables().PEX_ROOT) # A generator that emits sys.path elements def scrubbed_sys_path(): """Yields a scrubbed version of sys.path.""" for p in sys.path[:]: if not isinstance(p, str): yield p # Scrub any/all pex locations from sys.path. pp = pathlib.Path(p) if pex_root not in pp.parents: yield p def scrub_from_sys_modules(): """Yields keys of sys.modules as candidates for scrubbing/removal.""" for k, m in sys.modules.items(): if k in sys_modules_whitelist: continue if hasattr(m, '__file__') and m.__file__ is not None: mp = pathlib.Path(m.__file__) if pex_root in mp.parents: yield k def scrub_env(): # Replace sys.path with a scrubbed version. sys.path[:] = list(scrubbed_sys_path()) # Drop module cache references from sys.modules. modules_to_scrub = list(scrub_from_sys_modules()) for m in modules_to_scrub: del sys.modules[m] logger( 'Scrubbing sys.path and sys.modules in preparation for pex bootstrap\n' ) logger(f'sys.path contains {len(sys.path)} items, ' f'sys.modules contains {len(sys.modules)} keys\n') # Scrub environment. scrub_env() logger(f'sys.path now contains {len(sys.path)} items, ' f'sys.modules now contains {len(sys.modules)} keys\n')
def pex_python_paths(cls): """A list of paths to Python interpreter binaries as defined by a PEX_PYTHON_PATH defined in either in '/etc/pexrc', '~/.pexrc'. PEX_PYTHON_PATH defines a colon-seperated list of paths to interpreters that a pex can be built and ran against. :return: paths to interpreters as specified by PEX_PYTHON_PATH :rtype: list """ ppp = Variables.from_rc().get('PEX_PYTHON_PATH') if ppp: return ppp.split(os.pathsep) else: return []
def test_pex_vars_defaults_stripped(): # type: () -> None v = Variables(environ={}) # bool assert v.PEX_ALWAYS_CACHE is not None assert Variables.PEX_ALWAYS_CACHE.strip_default(v) is None # string assert v.PEX_PATH is not None assert Variables.PEX_PATH.strip_default(v) is None # int assert v.PEX_VERBOSE is not None assert Variables.PEX_VERBOSE.strip_default(v) is None
def test_pex_vars_value_or(tmpdir): # type: (Any) -> None v = Variables(environ={}) assert v.PEX_ROOT is not None, "Expected PEX_ROOT to be a defaulted variable." pex_root = str(tmpdir) assert pex_root == Variables.PEX_ROOT.value_or(v, pex_root) unwriteable_pex_root = os.path.join(pex_root, "unwriteable") os.mkdir(unwriteable_pex_root, 0o444) assert unwriteable_pex_root != Variables.PEX_ROOT.value_or( v, unwriteable_pex_root ), ("Expected the fallback to be validated, and in the case of PEX_ROOT, replaced with a " "writeable tmp dir")
def test_pex_bool_variables(): Variables(environ={})._get_bool("NOT_HERE", default=False) is False Variables(environ={})._get_bool("NOT_HERE", default=True) is True for value in ("0", "faLsE", "false"): for default in (True, False): Variables(environ={"HERE": value})._get_bool("HERE", default=default) is False for value in ("1", "TrUe", "true"): for default in (True, False): Variables(environ={"HERE": value})._get_bool("HERE", default=default) is True with pytest.raises(SystemExit): Variables(environ={"HERE": "garbage"})._get_bool("HERE") # end to end assert Variables().PEX_ALWAYS_CACHE is False assert Variables({"PEX_ALWAYS_CACHE": "1"}).PEX_ALWAYS_CACHE is True
def test_pex_bool_variables(): Variables(environ={})._get_bool('NOT_HERE', default=False) is False Variables(environ={})._get_bool('NOT_HERE', default=True) is True for value in ('0', 'faLsE', 'false'): for default in (True, False): Variables(environ={'HERE': value})._get_bool('HERE', default=default) is False for value in ('1', 'TrUe', 'true'): for default in (True, False): Variables(environ={'HERE': value})._get_bool('HERE', default=default) is True with pytest.raises(SystemExit): Variables(environ={'HERE': 'garbage'})._get_bool('HERE') # end to end assert Variables().PEX_ALWAYS_CACHE is False assert Variables({'PEX_ALWAYS_CACHE': '1'}).PEX_ALWAYS_CACHE is True
def test_pex_bool_variables(): # type: () -> None assert Variables(environ={})._maybe_get_bool("NOT_HERE") is None with pytest.raises(NoValueError): Variables(environ={})._get_bool("NOT_HERE") for value in ("0", "faLsE", "false"): assert Variables(environ={"HERE": value})._get_bool("HERE") is False for value in ("1", "TrUe", "true"): assert Variables(environ={"HERE": value})._get_bool("HERE") is True with pytest.raises(SystemExit): Variables(environ={"HERE": "garbage"})._get_bool("HERE") # end to end assert Variables().PEX_ALWAYS_CACHE is False assert Variables({"PEX_ALWAYS_CACHE": "1"}).PEX_ALWAYS_CACHE is True
def test_pex_root_unwriteable(): with temporary_dir() as td: pex_root = os.path.realpath(os.path.join(td, "pex_root")) os.mkdir(pex_root, 0o444) env = Variables(environ=dict(PEX_ROOT=pex_root)) with warnings.catch_warnings(record=True) as log: assert pex_root != env.PEX_ROOT assert 1 == len(log) message = log[0].message assert isinstance(message, PEXWarning) assert pex_root in str(message) assert env.PEX_ROOT in str(message) assert env.PEX_ROOT == env.PEX_ROOT, ( "When an ephemeral PEX_ROOT is materialized it should be stable.")
def test_from_env(): pex_root = os.path.realpath('/pex_root') environ = dict(PEX_ROOT=pex_root, PEX_MODULE='entry:point', PEX_SCRIPT='script.sh', PEX_FORCE_LOCAL='true', PEX_INHERIT_PATH='true', PEX_IGNORE_ERRORS='true', PEX_ALWAYS_CACHE='true') info = dict(pex_root=pex_root, entry_point='entry:point', script='script.sh', zip_safe=False, inherit_path=True, ignore_errors=True, always_write_cache=True) assert_same_info(PexInfo(info=info), PexInfo.from_env(env=Variables(environ=environ)))
def test_iter_help(): for variable_name, variable_type, variable_text in Variables.iter_help(): assert variable_name.startswith("PEX_") assert "\n" not in variable_type assert "\n" not in variable_text
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def test_pex_vars_set(): v = Variables(environ={}) v.set("HELLO", "42") assert v._get_int("HELLO") == 42 v.delete("HELLO") assert v._get_int("HELLO") is None
def test_pex_get_kv(): v = Variables(environ={}) assert v._get_kv("HELLO") is None assert v._get_kv("=42") is None assert v._get_kv("TOO=MANY=COOKS") is None assert v._get_kv("THIS=WORKS") == ["THIS", "WORKS"]
def test_pex_from_rc(): with tempfile.NamedTemporaryFile(mode="w") as pexrc: pexrc.write("HELLO=42") pexrc.flush() v = Variables(rc=pexrc.name) assert v._get_int("HELLO") == 42
def test_pexrc_precedence(): with tempfile.NamedTemporaryFile(mode="w") as pexrc: pexrc.write("HELLO=FORTYTWO") pexrc.flush() v = Variables(environ={"HELLO": 42}, rc=pexrc.name) assert v._get_int("HELLO") == 42
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') else: pex_python_path = "" interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def test_pexrc_precedence(): with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() v = Variables(rc=pexrc.name, environ={'HELLO': 42}) assert v._get_int('HELLO') == 42
def test_pex_from_rc(): with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=42') pexrc.flush() v = Variables(rc=pexrc.name) assert v._get_int('HELLO') == 42
def test_pex_get_kv(): v = Variables(environ={}) assert v._get_kv('HELLO') is None assert v._get_kv('=42') is None assert v._get_kv('TOO=MANY=COOKS') is None assert v._get_kv('THIS=WORKS') == ['THIS', 'WORKS']
def test_iter_help(): for variable_name, variable_type, variable_text in Variables.iter_help(): assert variable_name.startswith('PEX_') assert '\n' not in variable_type assert '\n' not in variable_text