def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def _add_test_hello_to_pex(ep):
with tempfile.NamedTemporaryFile() as tf:
tf.write(b'def hello(): print("hello")')
tf.flush()
pex_builder = PEXBuilder()
pex_builder.add_source(tf.name, 'test.py')
pex_builder.set_entry_point(ep)
pex_builder.freeze()
yield pex_builder
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
s1 = os.stat(src)
s2 = os.stat(os.path.join(path, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
s1 = os.stat(src)
s2 = os.stat(os.path.join(path, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO],
s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def write_simple_pex(td,
exe_contents=None,
dists=None,
sources=None,
coverage=False,
interpreter=None,
pex_info=None):
"""Write a pex file that optionally contains an executable entry point.
:param str td: temporary directory path
:param str exe_contents: entry point python file
:param dists: distributions to include, typically sdists or bdists
:type: list of :class:`pex.third_party.pkg_resources.Distribution`
:param sources: sources to include, as a list of pairs (env_filename, contents)
:type sources: list of (str, str)
:param bool coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:param pex_info: a custom PexInfo to use to build the pex.
:type pex_info: :class:`pex.pex_info.PexInfo`
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
pb = PEXBuilder(
path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter,
pex_info=pex_info,
)
for dist in dists:
pb.add_dist_location(
dist.location if isinstance(dist, Distribution) else dist)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, "w") as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
if exe_contents:
with open(os.path.join(td, "exe.py"), "w") as fp:
fp.write(exe_contents)
pb.set_executable(os.path.join(td, "exe.py"))
pb.freeze()
return pb
def write_simple_pex(
td, # type: str
exe_contents=None, # type: Optional[str]
dists=None, # type: Optional[Iterable[Distribution]]
sources=None, # type: Optional[Iterable[Tuple[str, str]]]
coverage=False, # type: bool
interpreter=None, # type: Optional[PythonInterpreter]
pex_info=None, # type: Optional[PexInfo]
):
# type: (...) -> PEXBuilder
"""Write a pex file that optionally contains an executable entry point.
:param td: temporary directory path
:param exe_contents: entry point python file
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
:param pex_info: a custom PexInfo to use to build the pex.
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
pb = PEXBuilder(
path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter,
pex_info=pex_info,
)
for dist in dists:
pb.add_dist_location(
dist.location if isinstance(dist, Distribution) else dist)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, "w") as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
if exe_contents:
with open(os.path.join(td, "exe.py"), "w") as fp:
fp.write(exe_contents)
pb.set_executable(os.path.join(td, "exe.py"))
pb.freeze()
return pb
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
path_clone = os.path.join(path, '__clone')
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def test_pex_builder_add_source_relpath_issues_1192(
tmp_chroot, # type: str
copy_mode, # type: CopyMode.Value
):
# type: (...) -> None
pb = PEXBuilder(copy_mode=copy_mode)
with safe_open("src/main.py", "w") as fp:
fp.write("import sys; sys.exit(42)")
pb.add_source("src/main.py", "main.py")
pb.set_entry_point("main")
pb.build("test.pex")
process = Executor.open_process(cmd=[os.path.abspath("test.pex")])
process.wait()
assert 42 == process.returncode
def _add_test_hello_to_pex(ep):
with temporary_dir() as td:
hello_file = "\n".join([
"def hello():",
" print('hello')",
])
with temporary_file(root_dir=td) as tf:
with open(tf.name, 'w') as handle:
handle.write(hello_file)
pex_builder = PEXBuilder()
pex_builder.add_source(tf.name, 'test.py')
pex_builder.set_entry_point(ep)
pex_builder.freeze()
yield pex_builder
def test_execute_interpreter_dashm_module():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
with temporary_file(root_dir=pex_chroot) as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.close()
pex_builder.add_source(fp.name, 'foo/bar.py')
pex_builder.freeze()
process = PEX(pex_chroot).run(args=['-m', 'foo.bar', 'one', 'two'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b'foo.bar one two\n' == stdout
assert b'' == stderr
def _hydrate_pex_file(self, hydrated_pex_file):
# We extract source files into a temporary directory before creating the pex.
td = tempfile.mkdtemp()
with open_zip(self) as zf:
# Populate the pex with the pinned requirements and distribution names & hashes.
bootstrap_info = PexInfo.from_json(zf.read("BOOTSTRAP-PEX-INFO"))
bootstrap_builder = PEXBuilder(pex_info=bootstrap_info,
interpreter=PythonInterpreter.get())
# Populate the pex with the needed code.
try:
ipex_info = json.loads(zf.read("IPEX-INFO").decode("utf-8"))
for path in ipex_info["code"]:
unzipped_source = zf.extract(path, td)
bootstrap_builder.add_source(
unzipped_source, env_filename=_strip_app_code_prefix(path))
except Exception as e:
raise ValueError(
"Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}".
format(e=e, info=json.dumps(ipex_info, indent=4)))
# Perform a fully pinned intransitive resolve to hydrate the install cache.
resolver_settings = ipex_info["resolver_settings"]
sanitized_requirements = _sanitize_requirements(
bootstrap_info.requirements)
bootstrap_info = modify_pex_info(bootstrap_info,
requirements=sanitized_requirements)
bootstrap_builder.info = bootstrap_info
resolved_distributions = resolver.resolve(
requirements=bootstrap_info.requirements,
cache=bootstrap_info.pex_root,
platform="current",
transitive=False,
interpreter=bootstrap_builder.interpreter,
**resolver_settings)
# TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from
# BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these
# requirements or recognize that they should be pulled from the cache for some reason.
for resolved_dist in resolved_distributions:
bootstrap_builder.add_distribution(resolved_dist.distribution)
bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False)
def build_and_check(path, copy):
# type: (str, bool) -> None
pb = PEXBuilder(path=path, copy=copy)
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO],
s1[stat.ST_DEV]) == (s2[stat.ST_INO],
s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def build_and_check(path, precompile):
pb = PEXBuilder(path)
pb.add_source(src, 'lib/src.py')
pb.set_executable(exe, 'exe.py')
pb.freeze(bytecode_compile=precompile)
for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
bootstrap_pycs = []
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
def write_simple_pex(td,
exe_contents,
dists=None,
sources=None,
coverage=False,
interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def build_and_check(path, precompile):
# type: (str, bool) -> None
pb = PEXBuilder(path=path)
pb.add_source(src, "lib/src.py")
pb.set_executable(exe, "exe.py")
pb.freeze(bytecode_compile=precompile)
for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, pb.info.bootstrap)
bootstrap_pycs = [] # type: List[str]
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith(".pyc"))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
def test_execute_interpreter_dashm_module():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.add_source(None, "foo/__init__.py")
with tempfile.NamedTemporaryFile() as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.flush()
pex_builder.add_source(fp.name, "foo/bar.py")
pex_builder.freeze()
pex = PEX(pex_chroot)
process = pex.run(
args=["-m", "foo.bar", "one", "two"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False,
)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b"foo.bar one two\n" == stdout
assert b"" == stderr
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex, interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def assert_access_zipped_assets(distribution_helper_import):
# type: (str) -> bytes
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with temporary_dir() as td1, temporary_dir() as td2:
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, "exe.py"), "w") as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, "my_package", "submodule")
safe_mkdir(submodule)
mod_path = os.path.join(submodule, "mod.py")
with open(mod_path, "w") as fp:
fp.write("accessed")
pb.add_source(fp.name, "my_package/submodule/mod.py")
pb.add_source(None, "my_package/__init__.py")
pb.add_source(None, "my_package/submodule/__init__.py")
pex = os.path.join(td2, "app.pex")
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b"accessed\n" == stdout
return cast(bytes, stderr)
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def build_and_check(copy_mode):
# type: (CopyMode.Value) -> None
pb = PEXBuilder(copy_mode=copy_mode)
path = pb.path()
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy_mode == CopyMode.COPY:
assert not is_link
else:
# Since os.stat follows symlinks; so in CopyMode.SYMLINK, this just proves
# the symlink points to the original file. Going further and checking path
# and path_clone for the presence of a symlink (an os.islink test) is
# trickier since a Linux hardlink of a symlink produces a symlink whereas a
# macOS hardlink of a symlink produces a hardlink.
assert is_link
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def build_pex(reqs, options, cache=None):
interpreters = None # Default to the current interpreter.
pex_python_path = options.python_path # If None, this will result in using $PATH.
# TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag.
if pex_python_path is None and (options.rc_file
or not ENV.PEX_IGNORE_RCFILES):
rc_variables = Variables(rc=options.rc_file)
pex_python_path = rc_variables.PEX_PYTHON_PATH
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed("Resolving interpreters", V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" %
full_path_or_basename)
return interp
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed("Resolving interpreters", V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
try:
interpreters = list(
iter_compatible_interpreters(
path=pex_python_path,
interpreter_constraints=constraints))
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Could not find a compatible interpreter."),
CANNOT_SETUP_INTERPRETER,
)
platforms = OrderedSet(options.platforms)
interpreters = interpreters or []
if options.platforms and options.resolve_local_platforms:
with TRACER.timed(
"Searching for local interpreters matching {}".format(
", ".join(map(str, platforms)))):
candidate_interpreters = OrderedSet(
iter_compatible_interpreters(path=pex_python_path))
candidate_interpreters.add(PythonInterpreter.get())
for candidate_interpreter in candidate_interpreters:
resolved_platforms = candidate_interpreter.supported_platforms.intersection(
platforms)
if resolved_platforms:
for resolved_platform in resolved_platforms:
TRACER.log("Resolved {} for platform {}".format(
candidate_interpreter, resolved_platform))
platforms.remove(resolved_platform)
interpreters.append(candidate_interpreter)
if platforms:
TRACER.log(
"Could not resolve a local interpreter for {}, will resolve only binary distributions "
"for {}.".format(
", ".join(map(str, platforms)),
"this platform"
if len(platforms) == 1 else "these platforms",
))
interpreter = (PythonInterpreter.latest_release_of_min_compatible_version(
interpreters) if interpreters else None)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
include_tools=options.include_tools or options.venv,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead.")
for directory in OrderedSet(options.sources_directory +
options.resources_directory):
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.unzip = options.unzip
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv
pex_info.venv_copies = options.venv_copies
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = InheritPath.for_value(options.inherit_path)
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
indexes = compute_indexes(options)
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(reqs +
options.requirement_files)):
if options.cache_ttl:
pex_warnings.warn(
"The --cache-ttl option is deprecated and no longer has any effect."
)
if options.headers:
pex_warnings.warn(
"The --header option is deprecated and no longer has any effect."
)
network_configuration = NetworkConfiguration(
retries=options.retries,
timeout=options.timeout,
proxy=options.proxy,
cert=options.cert,
client_cert=options.client_cert,
)
try:
if options.pex_repository:
with TRACER.timed("Resolving requirements from PEX {}.".format(
options.pex_repository)):
resolveds = resolve_from_pex(
pex=options.pex_repository,
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
network_configuration=network_configuration,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
manylinux=options.manylinux,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
indexes=indexes,
find_links=options.find_links,
resolver_version=ResolverVersion.for_value(
options.resolver_version),
network_configuration=network_configuration,
cache=cache,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors,
)
for resolved_dist in resolveds:
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(
resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(str(e))
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].items():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe',
action='store_false',
dest='zip_safe',
default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(options.python,
PythonInterpreter.from_binary(
options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
copy_package(pex_builder,
'pkg_resources',
prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
