def assert_chroot_perms(copyfn):
with temporary_dir() as src:
one = os.path.join(src, "one")
touch(one)
two = os.path.join(src, "two")
touch(two)
chmod_plus_x(two)
with temporary_dir() as dst:
chroot = Chroot(dst)
copyfn(chroot, one, "one")
copyfn(chroot, two, "two")
assert extract_perms(one) == extract_perms(
os.path.join(chroot.path(), "one"))
assert extract_perms(two) == extract_perms(
os.path.join(chroot.path(), "two"))
zip_path = os.path.join(src, "chroot.zip")
chroot.zip(zip_path)
with temporary_dir() as extract_dir:
with contextlib.closing(PermPreservingZipFile(zip_path)) as zf:
zf.extractall(extract_dir)
assert extract_perms(one) == extract_perms(
os.path.join(extract_dir, "one"))
assert extract_perms(two) == extract_perms(
os.path.join(extract_dir, "two"))
def build(self,
filename,
bytecode_compile=True,
deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
tmp_zip = filename + "~"
try:
os.unlink(tmp_zip)
self._logger.warning(
"Previous binary unexpectedly exists, cleaning: {}".format(
tmp_zip))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with safe_open(tmp_zip, "ab") as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes("{}\n".format(self._shebang)))
with TRACER.timed("Zipping PEX file."):
self._chroot.zip(tmp_zip,
mode="a",
deterministic_timestamp=deterministic_timestamp)
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmp_zip, filename)
chmod_plus_x(filename)
def build(self, filename, bytecode_compile=True, deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
try:
os.unlink(filename + '~')
self._logger.warning('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
if os.path.dirname(filename):
safe_mkdir(os.path.dirname(filename))
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes('%s\n' % self._shebang))
self._chroot.zip(filename + '~', mode='a', deterministic_timestamp=deterministic_timestamp)
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def build(self, filename, bytecode_compile=True, deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
try:
os.unlink(filename + '~')
self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
if os.path.dirname(filename):
safe_mkdir(os.path.dirname(filename))
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes('%s\n' % self._shebang))
self._chroot.zip(filename + '~', mode='a', deterministic_timestamp=deterministic_timestamp)
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def test_is_exe(temporary_working_dir):
# type: (str) -> None
touch("all_exe")
chmod_plus_x("all_exe")
assert is_exe("all_exe")
touch("other_exe")
os.chmod("other_exe", 0o665)
assert not is_exe("other_exe")
touch("not_exe")
assert not is_exe("not_exe")
os.mkdir("exe_dir")
chmod_plus_x("exe_dir")
assert not is_exe("exe_dir")
def zip_fixture():
with temporary_dir() as target_dir:
one = os.path.join(target_dir, "one")
touch(one)
two = os.path.join(target_dir, "two")
touch(two)
chmod_plus_x(two)
assert extract_perms(one) != extract_perms(two)
zip_file = os.path.join(target_dir, "test.zip")
with contextlib.closing(PermPreservingZipFile(zip_file, "w")) as zf:
zf.write(one, "one")
zf.write(two, "two")
yield zip_file, os.path.join(target_dir, "extract"), one, two
def zip_fixture():
with temporary_dir() as target_dir:
one = os.path.join(target_dir, 'one')
touch(one)
two = os.path.join(target_dir, 'two')
touch(two)
chmod_plus_x(two)
assert extract_perms(one) != extract_perms(two)
zip_file = os.path.join(target_dir, 'test.zip')
with contextlib.closing(PermPreservingZipFile(zip_file, 'w')) as zf:
zf.write(one, 'one')
zf.write(two, 'two')
yield zip_file, os.path.join(target_dir, 'extract'), one, two
def zip_fixture():
with temporary_dir() as target_dir:
one = os.path.join(target_dir, 'one')
touch(one)
two = os.path.join(target_dir, 'two')
touch(two)
chmod_plus_x(two)
assert extract_perms(one) != extract_perms(two)
zip_file = os.path.join(target_dir, 'test.zip')
with contextlib.closing(PermPreservingZipFile(zip_file, 'w')) as zf:
zf.write(one, 'one')
zf.write(two, 'two')
yield zip_file, os.path.join(target_dir, 'extract'), one, two
def build(self,
filename,
bytecode_compile=True,
deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
tmp_zip = filename + "~"
try:
os.unlink(tmp_zip)
self._logger.warning(
"Previous binary unexpectedly exists, cleaning: {}".format(
tmp_zip))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with safe_open(tmp_zip, "ab") as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes("{}\n".format(self._shebang)))
with TRACER.timed("Zipping PEX file."):
self._chroot.zip(
tmp_zip,
mode="a",
deterministic_timestamp=deterministic_timestamp,
# When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions
# as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since
# those installed wheels reside in a shared cache, they can be in-use by other
# processes and so their code may be in the process of being bytecode compiled as we
# attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary
# pyc files that we should avoid copying since they are unuseful and inherently
# racy.
exclude_file=is_pyc_temporary_file,
)
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmp_zip, filename)
chmod_plus_x(filename)
def assert_chroot_perms(copyfn):
with temporary_dir() as src:
one = os.path.join(src, 'one')
touch(one)
two = os.path.join(src, 'two')
touch(two)
chmod_plus_x(two)
with temporary_dir() as dst:
chroot = Chroot(dst)
copyfn(chroot, one, 'one')
copyfn(chroot, two, 'two')
assert extract_perms(one) == extract_perms(os.path.join(chroot.path(), 'one'))
assert extract_perms(two) == extract_perms(os.path.join(chroot.path(), 'two'))
zip_path = os.path.join(src, 'chroot.zip')
chroot.zip(zip_path)
with temporary_dir() as extract_dir:
with contextlib.closing(PermPreservingZipFile(zip_path)) as zf:
zf.extractall(extract_dir)
assert extract_perms(one) == extract_perms(os.path.join(extract_dir, 'one'))
assert extract_perms(two) == extract_perms(os.path.join(extract_dir, 'two'))
def populate_venv_with_pex(
venv, # type: Virtualenv
pex, # type: PEX
bin_path=BinPath.FALSE, # type: BinPath.Value
python=None, # type: Optional[str]
collisions_ok=True, # type: bool
):
# type: (...) -> str
venv_python = python or venv.interpreter.binary
venv_bin_dir = os.path.dirname(python) if python else venv.bin_dir
venv_dir = os.path.dirname(venv_bin_dir) if python else venv.venv_dir
# 1. Populate the venv with the PEX contents.
provenance = defaultdict(list)
def record_provenance(src_to_dst):
# type: (Iterable[Tuple[str, str]]) -> None
for src, dst in src_to_dst:
provenance[dst].append(src)
pex_info = pex.pex_info()
if zipfile.is_zipfile(pex.path()):
record_provenance(
PEXEnvironment(pex.path()).explode_code(venv.site_packages_dir,
exclude=("__main__.py", )))
else:
record_provenance(
_copytree(
src=pex.path(),
dst=venv.site_packages_dir,
exclude=(pex_info.internal_cache, pex_builder.BOOTSTRAP_DIR,
"__main__.py"),
))
for dist in pex.activate():
record_provenance(
_copytree(src=dist.location,
dst=venv.site_packages_dir,
exclude=("bin", )))
dist_bin_dir = os.path.join(dist.location, "bin")
if os.path.isdir(dist_bin_dir):
record_provenance(_copytree(dist_bin_dir, venv.bin_dir))
collisions = {
dst: srcs
for dst, srcs in provenance.items() if len(srcs) > 1
}
if collisions:
message_lines = [
"Encountered {collision} building venv at {venv_dir} from {pex}:".
format(collision=pluralize(collisions, "collision"),
venv_dir=venv_dir,
pex=pex.path())
]
for index, (dst, srcs) in enumerate(collisions.items(), start=1):
message_lines.append(
"{index}. {dst} was provided by:\n\t{srcs}".format(
index=index, dst=dst, srcs="\n\t".join(srcs)))
message = "\n".join(message_lines)
if not collisions_ok:
raise CollisionError(message)
pex_warnings.warn(message)
# 2. Add a __main__ to the root of the venv for running the venv dir like a loose PEX dir
# and a main.py for running as a script.
shebang = "#!{} -sE".format(venv_python)
main_contents = dedent("""\
{shebang}
if __name__ == "__main__":
import os
import sys
venv_dir = os.path.abspath(os.path.dirname(__file__))
venv_bin_dir = os.path.join(venv_dir, "bin")
shebang_python = {shebang_python!r}
python = os.path.join(venv_bin_dir, os.path.basename(shebang_python))
if sys.executable not in (python, shebang_python):
sys.stderr.write("Re-execing from {{}}\\n".format(sys.executable))
os.execv(python, [python, "-sE"] + sys.argv)
os.environ["VIRTUAL_ENV"] = venv_dir
sys.path.extend(os.environ.get("PEX_EXTRA_SYS_PATH", "").split(os.pathsep))
bin_path = os.environ.get("PEX_VENV_BIN_PATH", {bin_path!r})
if bin_path != "false":
PATH = os.environ.get("PATH", "").split(os.pathsep)
if bin_path == "prepend":
PATH.insert(0, venv_bin_dir)
elif bin_path == "append":
PATH.append(venv_bin_dir)
else:
sys.stderr.write(
"PEX_VENV_BIN_PATH must be one of 'false', 'prepend' or 'append', given: "
"{{!r}}\\n".format(
bin_path
)
)
sys.exit(1)
os.environ["PATH"] = os.pathsep.join(PATH)
PEX_EXEC_OVERRIDE_KEYS = ("PEX_INTERPRETER", "PEX_SCRIPT", "PEX_MODULE")
pex_overrides = {{
key: os.environ.pop(key) for key in PEX_EXEC_OVERRIDE_KEYS if key in os.environ
}}
if len(pex_overrides) > 1:
sys.stderr.write(
"Can only specify one of {{overrides}}; found: {{found}}\\n".format(
overrides=", ".join(PEX_EXEC_OVERRIDE_KEYS),
found=" ".join("{{}}={{}}".format(k, v) for k, v in pex_overrides.items())
)
)
sys.exit(1)
pex_script = pex_overrides.get("PEX_SCRIPT")
if pex_script:
script_path = os.path.join(venv_bin_dir, pex_script)
os.execv(script_path, [script_path] + sys.argv[1:])
pex_interpreter = pex_overrides.get("PEX_INTERPRETER", "").lower() in ("1", "true")
PEX_INTERPRETER_ENTRYPOINT = "code:interact"
entry_point = (
PEX_INTERPRETER_ENTRYPOINT
if pex_interpreter
else pex_overrides.get("PEX_MODULE", {entry_point!r} or PEX_INTERPRETER_ENTRYPOINT)
)
if entry_point == PEX_INTERPRETER_ENTRYPOINT and len(sys.argv) > 1:
args = sys.argv[1:]
arg = args[0]
if arg == "-m":
if len(args) < 2:
sys.stderr.write("Argument expected for the -m option\\n")
sys.exit(2)
entry_point = module = args[1]
sys.argv = args[1:]
# Fall through to entry_point handling below.
else:
filename = arg
sys.argv = args
if arg == "-c":
if len(args) < 2:
sys.stderr.write("Argument expected for the -c option\\n")
sys.exit(2)
filename = "-c <cmd>"
content = args[1]
sys.argv = ["-c"] + args[2:]
elif arg == "-":
content = sys.stdin.read()
else:
with open(arg) as fp:
content = fp.read()
ast = compile(content, filename, "exec", flags=0, dont_inherit=1)
globals_map = globals().copy()
globals_map["__name__"] = "__main__"
globals_map["__file__"] = filename
locals_map = globals_map
{exec_ast}
sys.exit(0)
module_name, _, function = entry_point.partition(":")
if not function:
import runpy
runpy.run_module(module_name, run_name="__main__")
else:
import importlib
module = importlib.import_module(module_name)
# N.B.: Functions may be hung off top-level objects in the module namespace,
# e.g.: Class.method; so we drill down through any attributes to the final function
# object.
namespace, func = module, None
for attr in function.split("."):
func = namespace = getattr(namespace, attr)
sys.exit(func())
""".format(
shebang=shebang,
shebang_python=venv_python,
bin_path=bin_path,
entry_point=pex_info.entry_point,
exec_ast=("exec ast in globals_map, locals_map"
if venv.interpreter.version[0] == 2 else
"exec(ast, globals_map, locals_map)"),
))
with open(venv.join_path("__main__.py"), "w") as fp:
fp.write(main_contents)
chmod_plus_x(fp.name)
os.symlink(os.path.basename(fp.name), venv.join_path("pex"))
# 3. Re-write any (console) scripts to use the venv Python.
for script in venv.rewrite_scripts(python=venv_python, python_args="-sE"):
TRACER.log("Re-writing {}".format(script))
return shebang
