def test_pex_from_rc():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=42")
pexrc.flush()
v = Variables(rc=pexrc.name)
assert v._get_int("HELLO") == 42
def test_pexrc_precedence():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=FORTYTWO")
pexrc.flush()
v = Variables(rc=pexrc.name, environ={"HELLO": "42"})
assert v._get_int("HELLO") == 42
def create(
cls,
venv_dir, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
force=False, # type: bool
):
# type: (...) -> Virtualenv
venv_dir = os.path.abspath(venv_dir)
safe_mkdir(venv_dir, clean=force)
interpreter = interpreter or PythonInterpreter.get()
if interpreter.is_venv:
base_interpreter = interpreter.resolve_base_interpreter()
TRACER.log(
"Ignoring enclosing venv {} and using its base interpreter {} to create venv at {}"
" instead.".format(interpreter.prefix, base_interpreter.binary,
venv_dir),
V=3,
)
interpreter = base_interpreter
if interpreter.version[0] >= 3:
interpreter.execute(args=["-m", "venv", "--without-pip", venv_dir])
else:
virtualenv_py = resource_string(__name__, "virtualenv_16.7.10_py")
with named_temporary_file(mode="wb") as fp:
fp.write(virtualenv_py)
fp.close()
interpreter.execute(args=[
fp.name, "--no-pip", "--no-setuptools", "--no-wheel",
venv_dir
], )
return cls(venv_dir)
def _do_entry_point_verification(self):
entry_point = self._pex_info.entry_point
ep_split = entry_point.split(":")
# a.b.c:m ->
# ep_module = 'a.b.c'
# ep_method = 'm'
# Only module is specified
if len(ep_split) == 1:
ep_module = ep_split[0]
import_statement = "import {}".format(ep_module)
elif len(ep_split) == 2:
ep_module = ep_split[0]
ep_method = ep_split[1]
import_statement = "from {} import {}".format(ep_module, ep_method)
else:
raise self.InvalidEntryPoint("Failed to parse: `{}`".format(entry_point))
with named_temporary_file() as fp:
fp.write(import_statement.encode("utf-8"))
fp.close()
retcode = self.run([fp.name], env={"PEX_INTERPRETER": "1"})
if retcode != 0:
raise self.InvalidEntryPoint(
"Invalid entry point: `{}`\n"
"Entry point verification failed: `{}`".format(entry_point, import_statement)
)
def test_pex_paths():
# type: () -> None
# Tests that PEX_PATH allows importing sources from the referenced pex.
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex1_path = os.path.join(temp_dir, "pex1")
write_simple_pex(
pex1_path,
sources=[
("foo_pkg/__init__.py", ""),
("foo_pkg/foo_module.py", 'def foo_func():\n return "42"'),
],
)
pex2_path = os.path.join(temp_dir, "pex2")
pex2 = write_simple_pex(
pex2_path,
"import sys; from bar_pkg.bar_module import bar_func; "
"sys.stdout.write(bar_func()); sys.exit(0)",
sources=[
("bar_pkg/__init__.py", ""),
(
"bar_pkg/bar_module.py",
"from foo_pkg.foo_module import foo_func\ndef bar_func():\n return foo_func()",
),
],
)
rc = PEX(pex2.path()).run(stdin=None, stdout=fake_stdout, env={"PEX_PATH": pex1_path})
assert rc == 0
fake_stdout.seek(0)
assert fake_stdout.read() == b"42"
def compile(self, root, relpaths):
# type: (str, Iterable[str]) -> List[Text]
"""Compiles the given python source files using this compiler's interpreter.
:param root: The root path all the source files are found under.
:param relpaths: The relative paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(
to_bytes(_COMPILER_MAIN % {
"root": root,
"relpaths": relpaths
},
encoding="utf-8"))
fp.flush()
try:
_, out, _ = self._interpreter.execute(args=[fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
"encountered %r during bytecode compilation.\nstderr was:\n%s\n"
% (e, e.stderr))
return cast("Text", out).splitlines()
def _do_entry_point_verification(self):
entry_point = self._pex_info.entry_point
ep_split = entry_point.split(':')
# a.b.c:m ->
# ep_module = 'a.b.c'
# ep_method = 'm'
# Only module is specified
if len(ep_split) == 1:
ep_module = ep_split[0]
import_statement = 'import {}'.format(ep_module)
elif len(ep_split) == 2:
ep_module = ep_split[0]
ep_method = ep_split[1]
import_statement = 'from {} import {}'.format(ep_module, ep_method)
else:
raise self.InvalidEntryPoint("Failed to parse: `{}`".format(entry_point))
with named_temporary_file() as fp:
fp.write(import_statement.encode('utf-8'))
fp.close()
retcode = self.run([fp.name], env={'PEX_INTERPRETER': '1'})
if retcode != 0:
raise self.InvalidEntryPoint('Invalid entry point: `{}`\n'
'Entry point verification failed: `{}`'
.format(entry_point, import_statement))
def compile(self, root, relpaths):
"""Compiles the given python source files using this compiler's interpreter.
:param string root: The root path all the source files are found under.
:param list relpaths: The realtive paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(
to_bytes(_COMPILER_MAIN % {
'root': root,
'relpaths': relpaths
},
encoding='utf-8'))
fp.flush()
try:
out, _ = Executor.execute(
[self._interpreter.binary, '-sE', fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
'encountered %r during bytecode compilation.\nstderr was:\n%s\n'
% (e, e.stderr))
return out.splitlines()
def test_rc_ignore():
# type: () -> None
with named_temporary_file(mode="w") as pexrc:
pexrc.write("HELLO=FORTYTWO")
pexrc.flush()
v = Variables(rc=pexrc.name, environ={"PEX_IGNORE_RCFILES": "True"})
assert "HELLO" not in v._environ
def temporary_filename():
"""Creates a temporary filename.
This is useful when you need to pass a filename to an API. Windows requires all
handles to a file be closed before deleting/renaming it, so this makes it a bit
simpler."""
with named_temporary_file() as fp:
fp.write(b'')
fp.close()
yield fp.name
def test_pex_interpreter():
with named_temporary_file() as fp:
fp.write(b"print('Hello world')")
fp.flush()
env = make_env(PEX_INTERPRETER=1)
so, rc = run_simple_pex_test("", args=(fp.name,), coverage=True, env=env)
assert so == b'Hello world\n'
assert rc == 0
def test_hash():
empty_hash = sha1().hexdigest()
with named_temporary_file() as fp:
fp.flush()
assert empty_hash == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
string = b'asdf' * 1024 * sha1().block_size + b'extra padding'
fp.write(string)
fp.flush()
assert sha1(string).hexdigest() == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
empty_hash = sha1()
fp.write(b'asdf')
fp.flush()
hash_output = CacheHelper.hash(fp.name, digest=empty_hash)
assert hash_output == empty_hash.hexdigest()
def temporary_filename():
"""Creates a temporary filename.
This is useful when you need to pass a filename to an API. Windows requires all
handles to a file be closed before deleting/renaming it, so this makes it a bit
simpler."""
with named_temporary_file() as fp:
fp.write(b'')
fp.close()
yield fp.name
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with open_zip(tf.name, 'r') as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash():
empty_hash = sha1().hexdigest()
with named_temporary_file() as fp:
fp.flush()
assert empty_hash == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
string = b'asdf' * 1024 * sha1().block_size + b'extra padding'
fp.write(string)
fp.flush()
assert sha1(string).hexdigest() == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
empty_hash = sha1()
fp.write(b'asdf')
fp.flush()
hash_output = CacheHelper.hash(fp.name, digest=empty_hash)
assert hash_output == empty_hash.hexdigest()
def test_hash_consistency():
for reverse in (False, True):
with temporary_content(CONTENT) as td:
dir_hash = CacheHelper.dir_hash(td)
with named_temporary_file() as tf:
write_zipfile(td, tf.name, reverse=reverse)
with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf:
zip_hash = CacheHelper.zip_hash(zf)
assert zip_hash == dir_hash
assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash
def test_hash():
# type: () -> None
empty_hash_digest = sha1().hexdigest()
with named_temporary_file() as fp:
fp.flush()
assert empty_hash_digest == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
string = b"asdf" * 1024 * sha1().block_size + b"extra padding"
fp.write(string)
fp.flush()
assert sha1(string).hexdigest() == CacheHelper.hash(fp.name)
with named_temporary_file() as fp:
empty_hash = sha1()
fp.write(b"asdf")
fp.flush()
hash_output = CacheHelper.hash(fp.name, digest=empty_hash)
assert hash_output == empty_hash.hexdigest()
def test_named_temporary_file():
# type: () -> None
with named_temporary_file() as fp:
name = fp.name
fp.write(b"hi")
fp.flush()
assert os.path.exists(name)
with open(name) as new_fp:
assert new_fp.read() == "hi"
assert not os.path.exists(name)
def test_named_temporary_file():
name = ''
with named_temporary_file() as fp:
name = fp.name
fp.write(b'hi')
fp.flush()
assert os.path.exists(name)
with open(name) as new_fp:
assert new_fp.read() == 'hi'
assert not os.path.exists(name)
def test_pex_interpreter():
with named_temporary_file() as fp:
fp.write(b"print('Hello world')")
fp.flush()
env = os.environ.copy()
env['PEX_INTERPRETER'] = '1'
so, rc = run_simple_pex_test("", args=(fp.name,), coverage=True, env=env)
assert so == b'Hello world\n'
assert rc == 0
def test_named_temporary_file():
name = ''
with named_temporary_file() as fp:
name = fp.name
fp.write(b'hi')
fp.flush()
assert os.path.exists(name)
with open(name) as new_fp:
assert new_fp.read() == 'hi'
assert not os.path.exists(name)
def test_pex_run():
# type: () -> None
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'import sys; sys.stdout.write("hello"); sys.stderr.write("hello"); sys.exit(0)',
)
rc = PEX(pex.path()).run(stdin=None, stdout=fake_stdout, stderr=fake_stdout)
assert rc == 0
fake_stdout.seek(0)
assert fake_stdout.read() == b"hellohello"
def test_urllib_context_utf8_encoding():
BYTES = b'this is a decoded utf8 string'
with named_temporary_file() as tf:
tf.write(BYTES)
tf.flush()
local_link = Link.wrap(tf.name)
# Trick UrllibContext into thinking this is a remote link
class MockUrllibContext(UrllibContext):
def open(self, link):
return super(MockUrllibContext, self).open(local_link)
context = MockUrllibContext()
assert context.content(Link.wrap('http://www.google.com')) == BYTES.decode(
UrllibContext.DEFAULT_ENCODING)
def test_urllib_context_utf8_encoding():
BYTES = b'this is a decoded utf8 string'
with named_temporary_file() as tf:
tf.write(BYTES)
tf.flush()
local_link = Link.wrap(tf.name)
# Trick UrllibContext into thinking this is a remote link
class MockUrllibContext(UrllibContext):
def open(self, link):
return super(MockUrllibContext, self).open(local_link)
context = MockUrllibContext()
assert context.content(Link.wrap('http://www.google.com')) == BYTES.decode(
UrllibContext.DEFAULT_ENCODING)
def test_requests_context():
context = RequestsContext(verify=False)
with make_url(BLOB, make_md5(BLOB)) as url:
assert context.read(Link.wrap(url)) == BLOB
with make_url(BLOB, make_md5(BLOB)) as url:
filename = context.fetch(Link.wrap(url))
with open(filename, 'rb') as fp:
assert fp.read() == BLOB
# test local reading
with named_temporary_file() as tf:
tf.write(b'goop')
tf.flush()
assert context.read(Link.wrap(tf.name)) == b'goop'
def test_requests_context():
context = RequestsContext(verify=False)
with make_url(BLOB, make_md5(BLOB)) as url:
assert context.read(Link.wrap(url)) == BLOB
with make_url(BLOB, make_md5(BLOB)) as url:
filename = context.fetch(Link.wrap(url))
with open(filename, 'rb') as fp:
assert fp.read() == BLOB
# test local reading
with named_temporary_file() as tf:
tf.write(b'goop')
tf.flush()
assert context.read(Link.wrap(tf.name)) == b'goop'
def test_venv_pex_interpreter_special_modes(create_pex_venv):
# type: (CreatePexVenv) -> None
venv = create_pex_venv()
# special mode execute module: -m module
returncode, stdout, stderr = execute_venv_pex_interpreter(venv, extra_args=["-m"])
assert 2 == returncode, stderr
assert "" == stdout
returncode, stdout, stderr = execute_venv_pex_interpreter(
venv, extra_args=["-m", "fabric", "--version"]
)
assert 0 == returncode, stderr
versions = parse_fabric_version_output(stdout)
assert FABRIC_VERSION == versions["Fabric"]
# special mode execute code string: -c <str>
returncode, stdout, stderr = execute_venv_pex_interpreter(venv, extra_args=["-c"])
assert 2 == returncode, stderr
assert "" == stdout
fabric_file_code = "import fabric, os; print(os.path.realpath(fabric.__file__))"
expected_fabric_file_path = expected_file_path(venv, "fabric")
returncode, stdout, stderr = execute_venv_pex_interpreter(
venv, extra_args=["-c", fabric_file_code]
)
assert 0 == returncode, stderr
assert expected_fabric_file_path == stdout.strip()
# special mode execute stdin: -
returncode, stdout, stderr = execute_venv_pex_interpreter(
venv, code=fabric_file_code, extra_args=["-"]
)
assert 0 == returncode, stderr
assert expected_fabric_file_path == stdout.strip()
# special mode execute python file: <py file name>
with named_temporary_file(prefix="code", suffix=".py", mode="w") as fp:
fp.write(fabric_file_code)
fp.close()
returncode, stdout, stderr = execute_venv_pex_interpreter(
venv, code=fabric_file_code, extra_args=[fp.name]
)
assert 0 == returncode, stderr
assert expected_fabric_file_path == stdout.strip()
def test_pex_run_extra_sys_path():
# type: () -> None
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir, 'import sys; sys.stdout.write(":".join(sys.path)); sys.exit(0)'
)
rc = PEX(pex.path()).run(
stdin=None,
stdout=fake_stdout,
stderr=None,
env={"PEX_EXTRA_SYS_PATH": "extra/syspath/entry1:extra/syspath/entry2"},
)
assert rc == 0
fake_stdout.seek(0)
syspath = fake_stdout.read().split(b":")
assert b"extra/syspath/entry1" in syspath
assert b"extra/syspath/entry2" in syspath
def create(
cls,
venv_dir, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
force=False, # type: bool
copies=False, # type: bool
):
# type: (...) -> Virtualenv
venv_dir = os.path.abspath(venv_dir)
safe_mkdir(venv_dir, clean=force)
interpreter = interpreter or PythonInterpreter.get()
if interpreter.is_venv:
base_interpreter = interpreter.resolve_base_interpreter()
TRACER.log(
"Ignoring enclosing venv {} and using its base interpreter {} to create venv at {}"
" instead.".format(interpreter.prefix, base_interpreter.binary,
venv_dir),
V=3,
)
interpreter = base_interpreter
if interpreter.version[
0] >= 3 and not interpreter.identity.interpreter == "PyPy":
# N.B.: PyPy3 comes equipped with a venv module but it does not seem to work.
args = ["-m", "venv", "--without-pip", venv_dir]
if copies:
args.append("--copies")
interpreter.execute(args=args)
else:
virtualenv_py = resource_string(__name__, "virtualenv_16.7.10_py")
with named_temporary_file(mode="wb") as fp:
fp.write(virtualenv_py)
fp.close()
args = [
fp.name, "--no-pip", "--no-setuptools", "--no-wheel",
venv_dir
]
if copies:
args.append("--always-copy")
interpreter.execute(args=args)
return cls(venv_dir)
def compile(self, root, relpaths):
"""Compiles the given python source files using this compiler's interpreter.
:param string root: The root path all the source files are found under.
:param list relpaths: The realtive paths from the `root` of the source files to compile.
:returns: A list of relative paths of the compiled bytecode files.
:raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
"""
with named_temporary_file() as fp:
fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8'))
fp.flush()
try:
out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name])
except Executor.NonZeroExit as e:
raise self.CompilationFailure(
'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr)
)
return out.splitlines()
def assert_isolation(self, inherit_path, expected_output):
# type: (Union[str, bool], str) -> None
env = dict(PYTHONPATH=self.pythonpath)
with named_temporary_file() as fake_stdout:
with temporary_dir() as temp_dir:
pex_builder = write_simple_pex(
temp_dir,
pex_info=self.pex_info(inherit_path),
dists=self.dists,
exe_contents=self.exe,
)
# Test the PEX.run API.
rc = PEX(pex_builder.path()).run(stdout=fake_stdout, env=env)
assert rc == 0
fake_stdout.seek(0)
assert expected_output == fake_stdout.read().decode("utf-8")
# Test direct PEX execution.
assert expected_output == subprocess.check_output(
[sys.executable, pex_builder.path()], env=env
).decode("utf-8")
def test_rc_ignore():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'PEX_IGNORE_RCFILES': 'True'})
assert 'HELLO' not in v._environ
def test_pexrc_precedence():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'HELLO': 42})
assert v._get_int('HELLO') == 42
def test_pex_from_rc():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=42')
pexrc.flush()
v = Variables(rc=pexrc.name)
assert v._get_int('HELLO') == 42
def test_pexrc_precedence():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'HELLO': 42})
assert v._get_int('HELLO') == 42
def test_pex_from_rc():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=42')
pexrc.flush()
v = Variables(rc=pexrc.name)
assert v._get_int('HELLO') == 42
def test_rc_ignore():
with named_temporary_file(mode='w') as pexrc:
pexrc.write('HELLO=FORTYTWO')
pexrc.flush()
v = Variables(rc=pexrc.name, environ={'PEX_IGNORE_RCFILES': 'True'})
assert 'HELLO' not in v._environ