def test_pex_builder_deterministic_timestamp():
pb = PEXBuilder()
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target, deterministic_timestamp=True)
with zipfile.ZipFile(target) as zf:
assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())
def assert_access_zipped_assets(distribution_helper_import):
# type: (str) -> bytes
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with temporary_dir() as td1, temporary_dir() as td2:
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, "exe.py"), "w") as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, "my_package", "submodule")
safe_mkdir(submodule)
mod_path = os.path.join(submodule, "mod.py")
with open(mod_path, "w") as fp:
fp.write("accessed")
pb.add_source(fp.name, "my_package/submodule/mod.py")
pb.add_source(None, "my_package/__init__.py")
pb.add_source(None, "my_package/submodule/__init__.py")
pex = os.path.join(td2, "app.pex")
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b"accessed\n" == stdout
return cast(bytes, stderr)
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def pack_in_pex(requirements: Dict[str, str], output: str) -> str:
"""
Pack current environment using a pex.
:param requirements: list of requirements (ex ['tensorflow==0.1.10'])
:param output: location of the pex
:return: destination of the archive, name of the pex
"""
requirements_to_install = format_requirements(requirements)
fetchers = []
if _criteo.is_criteo():
fetchers.append(PyPIFetcher(pypi_base=CRITEO_PYPI_URL))
fetchers.append(PyPIFetcher())
resolver_option_builder = ResolverOptionsBuilder(use_manylinux=True,
fetchers=fetchers)
resolvables = [
Resolvable.get(req, resolver_option_builder)
for req in requirements_to_install
]
pex_builder = PEXBuilder(copy=True)
try:
resolveds = resolve_multi(resolvables, use_manylinux=True)
for resolved in resolveds:
_logger.debug("Add requirement %s", resolved.distribution)
pex_builder.add_distribution(resolved.distribution)
pex_builder.add_requirement(resolved.requirement)
except (Unsatisfiable, Untranslateable):
_logger.exception('Cannot create pex')
raise
pex_builder.build(output)
return output
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir,
interpreter=interpreter,
pex_info=pex_info,
copy=True)
if binary_tgt.shebang:
self.context.log.info(
'Found Python binary target {} with customized shebang, using it: {}'
.format(binary_tgt.name, binary_tgt.shebang))
builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug(
'No customized shebang found for {}'.format(
binary_tgt.name))
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Add target's interpreter compatibility constraints to pex info.
if is_python_target(tgt):
for constraint in tgt.compatibility:
builder.add_interpreter_constraint(constraint)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
build_for_current_platform_only_check(self.context.targets())
dump_requirement_libs(builder,
interpreter,
req_tgts,
self.context.log,
platforms=binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir,
'{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex, interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def test_pex_builder_deterministic_timestamp():
pb = PEXBuilder()
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target, deterministic_timestamp=True)
with zipfile.ZipFile(target) as zf:
assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def test_pex_builder_shebang():
pb = PEXBuilder()
pb.set_shebang('foobar')
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target)
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
def test_pex_builder_shebang():
pb = PEXBuilder()
pb.set_shebang('foobar')
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target)
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
def test_pex_executable():
# type: () -> None
# Tests that pex keeps executable permissions
with temporary_dir() as temp_dir:
pex_dir = os.path.join(temp_dir, "pex_dir")
safe_mkdir(pex_dir)
with open(os.path.join(pex_dir, "exe.py"), "w") as fp:
fp.write(
textwrap.dedent(
"""
import subprocess
import os
import sys
import my_package
path = os.path.join(os.path.dirname(my_package.__file__), 'bin/start.sh')
sys.stdout.write(subprocess.check_output([path]).decode('utf-8'))
"""
)
)
project_content = {
"setup.py": textwrap.dedent(
"""
from setuptools import setup
setup(
name='my_project',
version='0.0.0.0',
zip_safe=True,
packages=['my_package'],
package_data={'my_package': ['bin/*']},
install_requires=[],
)
"""
),
"my_package/__init__.py": 0,
"my_package/bin/start.sh": (
"#!/usr/bin/env bash\n" "echo 'hello world from start.sh!'"
),
"my_package/my_module.py": 'def do_something():\n print("hello world!")\n',
} # type: Dict[str, Union[str, int]]
pex_builder = PEXBuilder(path=pex_dir)
with temporary_content(project_content, perms=0o755) as project_dir:
installer = WheelBuilder(project_dir)
bdist = installer.bdist()
pex_builder.add_dist_location(bdist)
pex_builder.set_executable(os.path.join(pex_dir, "exe.py"))
pex_builder.freeze()
app_pex = os.path.join(os.path.join(temp_dir, "out_pex_dir"), "app.pex")
pex_builder.build(app_pex)
std_out, rc = run_simple_pex(app_pex, env={"PEX_ROOT": os.path.join(temp_dir, ".pex")})
assert rc == 0
assert std_out.decode("utf-8") == "hello world from start.sh!\n"
def test_pex_executable():
# Tests that pex keeps executable permissions
with temporary_dir() as temp_dir:
pex_dir = os.path.join(temp_dir, 'pex_dir')
safe_mkdir(pex_dir)
with open(os.path.join(pex_dir, 'exe.py'), 'w') as fp:
fp.write(
textwrap.dedent('''
import subprocess
import os
import sys
import my_package
path = os.path.join(os.path.dirname(my_package.__file__), 'bin/start.sh')
sys.stdout.write(subprocess.check_output([path]).decode('utf-8'))
'''))
project_content = {
'setup.py':
textwrap.dedent('''
from setuptools import setup
setup(
name='my_project',
version='0.0.0.0',
zip_safe=True,
packages=['my_package'],
package_data={'my_package': ['bin/*']},
install_requires=[],
)
'''),
'my_package/__init__.py':
0,
'my_package/bin/start.sh': ("#!/usr/bin/env bash\n"
"echo 'hello world from start.sh!'"),
'my_package/my_module.py':
'def do_something():\n print("hello world!")\n',
}
pex_builder = PEXBuilder(path=pex_dir)
with temporary_content(project_content, perms=0o755) as project_dir:
installer = WheelInstaller(project_dir)
bdist = DistributionHelper.distribution_from_path(
installer.bdist())
pex_builder.add_dist_location(bdist.location)
pex_builder.set_executable(os.path.join(pex_dir, 'exe.py'))
pex_builder.freeze()
app_pex = os.path.join(os.path.join(temp_dir, 'out_pex_dir'),
'app.pex')
pex_builder.build(app_pex)
std_out, rc = run_simple_pex(
app_pex, env={'PEX_ROOT': os.path.join(temp_dir, '.pex')})
assert rc == 0
assert std_out.decode('utf-8') == 'hello world from start.sh!\n'
def _build(self):
target_path = os.path.join(self.install_directory,
self.target.get("executable_name"))
parser, resolver_options_builder = configure_clp()
pex_builder = PEXBuilder(interpreter=self._get_interpreter(parser))
pex_builder.set_entry_point(self.target.get("entry_point"))
for dist in self._get_distributions(resolver_options_builder):
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.build(target_path)
self.directory.symlink_to_bin(self.target.get("executable_name"),
target_path)
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
if binary_tgt.shebang:
self.context.log.info('Found Python binary target {} with customized shebang, using it: {}'
.format(binary_tgt.name, binary_tgt.shebang))
builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug('No customized shebang found for {}'.format(binary_tgt.name))
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Add target's interpreter compatibility constraints to pex info.
if is_python_target(tgt):
for constraint in tgt.compatibility:
builder.add_interpreter_constraint(constraint)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
self._python_native_code_settings.check_build_for_current_platform_only(self.context.targets())
dump_requirement_libs(builder, interpreter, req_tgts, self.context.log,
platforms=binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def _build(self):
target_path = os.path.join(self.install_directory,
self.target.get("executable_name"))
parser, resolver_options_builder = configure_clp()
pex_builder = PEXBuilder(
interpreter=self._get_interpreter(parser)
)
pex_builder.set_entry_point(self.target.get("entry_point"))
for dist in self._get_distributions(resolver_options_builder):
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.build(target_path)
self.directory.symlink_to_bin(self.target.get("executable_name"),
target_path)
def pack_in_pex(requirements: List[str],
output: str,
ignored_packages: Collection[str] = [],
pex_inherit_path: str = "prefer",
editable_requirements: Dict[str, str] = {}
) -> str:
"""
Pack current environment using a pex.
:param requirements: list of requirements (ex {'tensorflow': '1.15.0'})
:param output: location of the pex
:param ignored_packages: packages to be exluded from pex
:param pex_inherit_path: see https://github.com/pantsbuild/pex/blob/master/pex/bin/pex.py#L264,
possible values ['false', 'fallback', 'prefer']
:return: destination of the archive, name of the pex
"""
interpreter = PythonInterpreter.get()
pex_info = PexInfo.default(interpreter)
pex_info.inherit_path = pex_inherit_path
pex_builder = PEXBuilder(
copy=True,
interpreter=interpreter,
pex_info=pex_info)
for current_package in editable_requirements.values():
_logger.debug("Add current path as source", current_package)
_walk_and_do(pex_builder.add_source, current_package)
try:
resolveds = resolve_multi(
requirements=requirements,
indexes=[CRITEO_PYPI_URL] if _is_criteo() else None)
for resolved in resolveds:
if resolved.distribution.key in ignored_packages:
_logger.debug(f"Ignore requirement {resolved.distribution}")
continue
else:
_logger.debug(f"Add requirement {resolved.distribution}")
pex_builder.add_distribution(resolved.distribution)
pex_builder.add_requirement(resolved.requirement)
except (Unsatisfiable, Untranslatable):
_logger.exception('Cannot create pex')
raise
pex_builder.build(output)
return output
def test_pex_builder_add_source_relpath_issues_1192(
tmp_chroot, # type: str
copy_mode, # type: CopyMode.Value
):
# type: (...) -> None
pb = PEXBuilder(copy_mode=copy_mode)
with safe_open("src/main.py", "w") as fp:
fp.write("import sys; sys.exit(42)")
pb.add_source("src/main.py", "main.py")
pb.set_entry_point("main")
pb.build("test.pex")
process = Executor.open_process(cmd=[os.path.abspath("test.pex")])
process.wait()
assert 42 == process.returncode
def write_and_run_simple_pex(inheriting=False):
"""Write a pex file that contains an executable entry point
:param inheriting: whether this pex should inherit site-packages paths
:type inheriting: bool
"""
with temporary_dir() as td:
pex_path = os.path.join(td, 'show_path.pex')
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write('') # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
pb.build(pex_path)
yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0]
def _hydrate_pex_file(self, hydrated_pex_file):
# We extract source files into a temporary directory before creating the pex.
td = tempfile.mkdtemp()
with open_zip(self) as zf:
# Populate the pex with the pinned requirements and distribution names & hashes.
bootstrap_info = PexInfo.from_json(zf.read("BOOTSTRAP-PEX-INFO"))
bootstrap_builder = PEXBuilder(pex_info=bootstrap_info,
interpreter=PythonInterpreter.get())
# Populate the pex with the needed code.
try:
ipex_info = json.loads(zf.read("IPEX-INFO").decode("utf-8"))
for path in ipex_info["code"]:
unzipped_source = zf.extract(path, td)
bootstrap_builder.add_source(
unzipped_source, env_filename=_strip_app_code_prefix(path))
except Exception as e:
raise ValueError(
"Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}".
format(e=e, info=json.dumps(ipex_info, indent=4)))
# Perform a fully pinned intransitive resolve to hydrate the install cache.
resolver_settings = ipex_info["resolver_settings"]
sanitized_requirements = _sanitize_requirements(
bootstrap_info.requirements)
bootstrap_info = modify_pex_info(bootstrap_info,
requirements=sanitized_requirements)
bootstrap_builder.info = bootstrap_info
resolved_distributions = resolver.resolve(
requirements=bootstrap_info.requirements,
cache=bootstrap_info.pex_root,
platform="current",
transitive=False,
interpreter=bootstrap_builder.interpreter,
**resolver_settings)
# TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from
# BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these
# requirements or recognize that they should be pulled from the cache for some reason.
for resolved_dist in resolved_distributions:
bootstrap_builder.add_distribution(resolved_dist.distribution)
bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False)
def write_and_run_simple_pex(inheriting):
# type: (str) -> Iterator[Text]
"""Write a pex file that contains an executable entry point.
:param inheriting: whether this pex should inherit site-packages paths.
"""
with temporary_dir() as td:
pex_path = os.path.join(td, "show_path.pex")
with open(os.path.join(td, "exe.py"), "w") as fp:
fp.write(u"") # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, "exe.py"))
pb.freeze()
pb.build(pex_path)
stdout, _ = run_simple_pex(pex_path, env={"PEX_VERBOSE": "1"})
yield stdout.decode("utf-8")
def test_venv_symlinked_source_issues_1239(tmpdir):
# type: (Any) -> None
src = os.path.join(str(tmpdir), "src")
main = os.path.join(src, "main.py")
with safe_open(main, "w") as fp:
fp.write("import sys; sys.exit(42)")
pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK)
pex_builder.set_executable(main)
pex_file = os.path.join(str(tmpdir), "a.pex")
pex_builder.build(pex_file, bytecode_compile=False)
assert 42 == subprocess.Popen(args=[pex_file]).wait()
venv = os.path.join(str(tmpdir), "a.venv")
subprocess.check_call(
args=[sys.executable, "-m", "pex.tools", pex_builder.path(), "venv", venv]
)
venv_pex = os.path.join(venv, "pex")
shutil.rmtree(src)
assert 42 == subprocess.Popen(args=[venv_pex]).wait()
def bootstrap(self, interpreter, pex_file_path, extra_reqs=None):
# Caching is done just by checking if the file at the specified path is already executable.
if not is_executable(pex_file_path):
pex_info = PexInfo.default(interpreter=interpreter)
if self.entry_point is not None:
pex_info.entry_point = self.entry_point
with safe_concurrent_creation(pex_file_path) as safe_path:
builder = PEXBuilder(interpreter=interpreter,
pex_info=pex_info)
all_reqs = list(self.base_requirements) + list(extra_reqs
or [])
dump_requirements(builder,
interpreter,
all_reqs,
logger,
platforms=['current'])
builder.build(safe_path)
return PEX(pex_file_path, interpreter)
def test_pex_builder_preamble():
# type: () -> None
with temporary_dir() as td:
target = os.path.join(td, "foo.pex")
should_create = os.path.join(td, "foo.1")
tempfile_preamble = "\n".join(
["import sys", "open('{0}', 'w').close()".format(should_create), "sys.exit(3)"]
)
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir,
interpreter=interpreter,
pex_info=pex_info,
copy=True)
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
dump_requirements(builder, interpreter, req_tgts, self.context.log,
binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir,
'{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def test_pex_builder_preamble():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
should_create = os.path.join(td, 'foo.1')
tempfile_preamble = "\n".join([
"import sys",
"open('{0}', 'w').close()".format(should_create),
"sys.exit(3)"
])
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def test_prex_builder_script_from_pex_path(tmpdir):
# type: (Any) -> None
pex_with_script = os.path.join(str(tmpdir), "script.pex")
with built_wheel(
name="my_project",
entry_points={
"console_scripts":
["my_app = my_project.my_module:do_something"]
},
) as my_whl:
pb = PEXBuilder()
pb.add_dist_location(my_whl)
pb.build(pex_with_script)
pex_file = os.path.join(str(tmpdir), "app.pex")
pb = PEXBuilder()
pb.info.pex_path = pex_with_script
pb.set_script("my_app")
pb.build(pex_file)
assert "hello world!\n" == subprocess.check_output(
args=[pex_file]).decode("utf-8")
def assert_force_local_implicit_ns_packages_issues_598(interpreter=None,
requirements=(),
create_ns_packages=True):
def create_foo_bar_setup(name, **extra_args):
setup_args = dict(
name=name,
version='0.0.1',
packages=['foo', 'foo.bar']
)
if create_ns_packages:
setup_args.update(namespace_packages=['foo', 'foo.bar'])
if requirements:
setup_args.update(install_requires=list(requirements))
setup_args.update(extra_args)
return dedent("""
from setuptools import setup
setup(**{setup_args!r})
""".format(setup_args=setup_args))
def with_foo_bar_ns_packages(content):
ns_packages = {
os.path.join(pkg, '__init__.py'): '__import__("pkg_resources").declare_namespace(__name__)'
for pkg in ('foo', 'foo/bar')
} if create_ns_packages else {}
ns_packages.update(content)
return ns_packages
content1 = with_foo_bar_ns_packages({
'foo/bar/spam.py': 'identify = lambda: 42',
'setup.py': create_foo_bar_setup('foo-bar-spam')
})
content2 = with_foo_bar_ns_packages({
'foo/bar/eggs.py': dedent("""
# NB: This only works when this content is unpacked loose on the filesystem!
def read_self():
with open(__file__) as fp:
return fp.read()
""")
})
content3 = with_foo_bar_ns_packages({
'foobaz': dedent("""\
#!python
import sys
from foo.bar import baz
sys.exit(baz.main())
"""),
'foo/bar/baz.py': dedent("""
import sys
from foo.bar import eggs, spam
def main():
assert len(eggs.read_self()) > 0
return spam.identify()
"""),
'setup.py': create_foo_bar_setup('foo-bar-baz', scripts=['foobaz'])
})
def add_requirements(builder, cache):
for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter):
builder.add_requirement(resolved_dist.requirement)
builder.add_distribution(resolved_dist.distribution)
def add_wheel(builder, content):
with temporary_content(content) as project:
dist = WheelInstaller(project, interpreter=builder.interpreter).bdist()
builder.add_dist_location(dist)
def add_sources(builder, content):
with temporary_content(content) as project:
for path in content.keys():
builder.add_source(os.path.join(project, path), path)
with nested(temporary_dir(), temporary_dir()) as (root, cache):
pex_info1 = PexInfo.default()
pex_info1.zip_safe = False
pex1 = os.path.join(root, 'pex1.pex')
builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1)
add_requirements(builder1, cache)
add_wheel(builder1, content1)
add_sources(builder1, content2)
builder1.build(pex1)
pex_info2 = PexInfo.default()
pex_info2.pex_path = pex1
pex2 = os.path.join(root, 'pex2')
builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2)
add_requirements(builder2, cache)
add_wheel(builder2, content3)
builder2.set_script('foobaz')
builder2.freeze()
assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE='9'))
class PexBuilderWrapper:
"""Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities."""
class Factory(Subsystem):
options_scope = "pex-builder-wrapper"
@classmethod
def register_options(cls, register):
super(PexBuilderWrapper.Factory, cls).register_options(register)
# TODO: make an analogy to cls.register_jvm_tool that can be overridden for python subsystems
# by a python_requirement_library() target, not just via pants.ini!
register(
"--setuptools-version",
advanced=True,
default="40.6.3",
fingerprint=True,
help=
"The setuptools version to include in the pex if namespace packages need "
"to be injected.",
)
register(
"--pex-version",
advanced=True,
default=pex_version,
fingerprint=True,
help="The pex version to include in any generated ipex files. "
"NOTE: This should ideally be the same as the pex version which pants "
f"itself depends on, which right now is {pex_version}.",
)
@classmethod
def subsystem_dependencies(cls):
return super(PexBuilderWrapper.Factory,
cls).subsystem_dependencies() + (
PythonRepos,
PythonSetup,
)
@classmethod
def create(cls, builder, log=None, generate_ipex=False):
options = cls.global_instance().get_options()
setuptools_requirement = f"setuptools=={options.setuptools_version}"
pex_requirement = f"pex=={options.pex_version}"
log = log or logging.getLogger(__name__)
return PexBuilderWrapper(
builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(
setuptools_requirement),
pex_requirement=PythonRequirement(pex_requirement),
log=log,
generate_ipex=generate_ipex,
)
def __init__(
self,
builder: PEXBuilder,
python_repos_subsystem: PythonRepos,
python_setup_subsystem: PythonSetup,
setuptools_requirement: PythonRequirement,
pex_requirement: PythonRequirement,
log,
generate_ipex: bool = False,
):
assert log is not None
self._builder = builder
self._python_repos_subsystem = python_repos_subsystem
self._python_setup_subsystem = python_setup_subsystem
self._setuptools_requirement = setuptools_requirement
self._pex_requirement = pex_requirement
self._log = log
self._distributions: Dict[str, Distribution] = {}
self._frozen = False
self._generate_ipex = generate_ipex
# If we generate a .ipex, we need to ensure all the code we copy into the underlying PEXBuilder
# is also added to the new PEXBuilder created in `._shuffle_original_build_info_into_ipex()`.
self._all_added_sources_resources: List[Path] = []
# If we generate a dehydrated "ipex" file, we need to make sure that it is aware of any special
# find_links repos attached to any single requirement, so it can later resolve those
# requirements when it is first bootstrapped, using the same resolve options.
self._all_find_links: OrderedSet[str] = OrderedSet()
def add_requirement_libs_from(self, req_libs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
reqs = [req for req_lib in req_libs for req in req_lib.requirements]
self.add_resolved_requirements(reqs, platforms=platforms)
class SingleDistExtractionError(Exception):
pass
def extract_single_dist_for_current_platform(self, reqs,
dist_key) -> Distribution:
"""Resolve a specific distribution from a set of requirements matching the current platform.
:param list reqs: A list of :class:`PythonRequirement` to resolve.
:param str dist_key: The value of `distribution.key` to match for a `distribution` from the
resolved requirements.
:return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
:raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the
given `dist_key`.
"""
distributions = self.resolve_distributions(reqs, platforms=["current"])
try:
matched_dist = assert_single_element(
dist for dists in distributions.values() for dist in dists
if dist.key == dist_key)
except (StopIteration, ValueError) as e:
raise self.SingleDistExtractionError(
f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
)
return matched_dist
def resolve_distributions(
self,
reqs: List[PythonRequirement],
platforms: Optional[List[Platform]] = None,
) -> Dict[str, List[Distribution]]:
"""Multi-platform dependency resolution.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param platforms: A list of platform strings to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:returns: A tuple `(map, transitive_reqs)`, where `map` is a dict mapping distribution name
to a list of resolved distributions, and `reqs` contains all transitive ==
requirements
needed to resolve the initial given requirements `reqs` for the given platforms.
"""
deduped_reqs = OrderedSet(reqs)
find_links: OrderedSet[str] = OrderedSet()
for req in deduped_reqs:
self._log.debug(f" Dumping requirement: {req}")
self._builder.add_requirement(str(req.requirement))
if req.repository:
find_links.add(req.repository)
# Resolve the requirements into distributions.
distributions = self._resolve_multi(
self._builder.interpreter,
list(deduped_reqs),
platforms,
list(find_links),
)
return distributions
def add_resolved_requirements(
self,
reqs: List[PythonRequirement],
platforms: Optional[List[Platform]] = None,
override_ipex_build_do_actually_add_distribution: bool = False,
) -> None:
"""Multi-platform dependency resolution for PEX files.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:param bool override_ipex_build_do_actually_add_distribution: When this PexBuilderWrapper is configured with
generate_ipex=True, this method won't add any distributions to
the output pex. The internal implementation of this class adds a
pex dependency to the output ipex file, and therefore needs to
override the default behavior of this method.
"""
distributions = self.resolve_distributions(reqs, platforms=platforms)
locations: Set[str] = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
if self._generate_ipex and not override_ipex_build_do_actually_add_distribution:
self._log.debug(
f" *AVOIDING* dumping distribution into ipex: .../{os.path.basename(dist.location)}"
)
self._register_distribution(dist)
else:
self._log.debug(
f" Dumping distribution: .../{os.path.basename(dist.location)}"
)
self.add_distribution(dist)
locations.add(dist.location)
def _resolve_multi(
self,
interpreter: PythonInterpreter,
requirements: List[PythonRequirement],
platforms: Optional[List[Platform]],
find_links: Optional[List[str]],
) -> Dict[str, List[Distribution]]:
"""Multi-platform dependency resolution for PEX files.
Returns a tuple containing a list of distributions that must be included in order to satisfy a
set of requirements, and the transitive == requirements for those distributions. This may
involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = list(find_links) if find_links else []
find_links.extend(python_repos.repos)
# Individual requirements from pants may have a `repository` link attached to them, which is
# extracted in `self.resolve_distributions()`. When generating a .ipex file with
# `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it
# tries to resolve all the requirements from BOOTSTRAP-PEX-INFO.
self._all_find_links.update(OrderedSet(find_links))
distributions: Dict[str, List[Distribution]] = defaultdict(list)
for platform in platforms:
requirements_cache_dir = os.path.join(
python_setup.resolver_cache_dir, str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
platform=platform,
indexes=python_repos.indexes,
find_links=find_links,
cache=requirements_cache_dir,
allow_prereleases=python_setup.resolver_allow_prereleases,
manylinux=python_setup.manylinux,
)
for resolved_dist in resolved_dists:
distributions[platform].append(resolved_dist.distribution)
return distributions
def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]:
buildroot = get_buildroot()
def get_chroot_path(relpath: str) -> str:
if type(tgt) == Files:
# Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package
# structure and so we chroot them relative to the build root so that they can be accessed
# via the normal Python filesystem APIs just as they would be accessed outside the
# chrooted environment. NB: This requires we mark the pex as not zip safe so
# these `Files` can still be accessed in the context of a built pex distribution.
self._builder.info.zip_safe = False
return relpath
return str(Path(relpath).relative_to(tgt.target_base))
def dump_source(relpath: str) -> None:
source_path = str(Path(buildroot, relpath))
dest_path = get_chroot_path(relpath)
self._all_added_sources_resources.append(Path(dest_path))
if has_resources(tgt):
self._builder.add_resource(filename=source_path,
env_filename=dest_path)
else:
self._builder.add_source(filename=source_path,
env_filename=dest_path)
return dump_source
def add_sources_from(self, tgt: Target) -> None:
dump_source = self._create_source_dumper(tgt)
self._log.debug(f" Dumping sources: {tgt}")
for relpath in tgt.sources_relative_to_buildroot():
try:
dump_source(relpath)
except OSError:
self._log.error(
f"Failed to copy {relpath} for target {tgt.address.spec}")
raise
if getattr(tgt, "_resource_target_specs", None) or getattr(
tgt, "_synthetic_resources_target", None):
# No one should be on old-style resources any more. And if they are,
# switching to the new python pipeline will be a great opportunity to fix that.
raise TaskError(
f"Old-style resources not supported for target {tgt.address.spec}. Depend on resources() "
"targets instead.")
def _prepare_inits(self) -> Set[str]:
chroot = self._builder.chroot()
sources = chroot.get("source") | chroot.get("resource")
missing_init_files = identify_missing_init_files(sources)
if missing_init_files:
with temporary_file(permissions=0o644) as ns_package:
ns_package.write(
b'__import__("pkg_resources").declare_namespace(__name__) # type: ignore[attr-defined]'
)
ns_package.flush()
for missing_init_file in missing_init_files:
self._all_added_sources_resources.append(
Path(missing_init_file))
self._builder.add_source(filename=ns_package.name,
env_filename=missing_init_file)
return missing_init_files
def set_emit_warnings(self, emit_warnings):
self._builder.info.emit_warnings = emit_warnings
def _set_major_minor_interpreter_constraint_for_ipex(
self,
info: PexInfo,
identity: PythonIdentity,
) -> PexInfo:
interpreter_name = identity.requirement.name
major, minor, _patch = identity.version
major_minor_only_constraint = f"{interpreter_name}=={major}.{minor}.*"
return ipex_launcher.modify_pex_info(
info, interpreter_constraints=[str(major_minor_only_constraint)])
def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
"""Replace the original builder with a new one, and just pull files from the old chroot."""
# Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is
# (the exact same interpreter we used to resolve those requirements here). This is the only (?)
# way to ensure that the ipex bootstrap uses the *exact* same interpreter version.
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info, self._builder.interpreter.identity)
# Remove all the original top-level requirements in favor of the transitive == requirements.
self._builder.info = ipex_launcher.modify_pex_info(self._builder.info,
requirements=[])
transitive_reqs = [
dist.as_requirement() for dist in self._distributions.values()
]
self.add_direct_requirements(transitive_reqs)
orig_info = self._builder.info.copy()
orig_chroot = self._builder.chroot()
# Mutate the PexBuilder object which is manipulated by this subsystem.
self._builder = PEXBuilder(interpreter=self._builder.interpreter)
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info, self._builder.interpreter.identity)
self._distributions = {}
return (orig_info, Path(orig_chroot.path()))
def _shuffle_original_build_info_into_ipex(self):
"""Create a "dehydrated" ipex file without any of its requirements, and specify that in two.
*-INFO files.
See ipex_launcher.py for details of how these files are used.
"""
orig_pex_info, orig_chroot = self._shuffle_underlying_pex_builder()
# Gather information needed to create IPEX-INFO.
all_code = [str(src) for src in self._all_added_sources_resources]
prefixed_code_paths = [
os.path.join(ipex_launcher.APP_CODE_PREFIX, src)
for src in all_code
]
for src, prefixed in zip(all_code, prefixed_code_paths):
# NB: Need to add under 'source' label for `self._prepare_inits()` to pick it up!
self._builder.chroot().copy(os.path.join(str(orig_chroot), src),
prefixed,
label="source")
python_repos = self._python_repos_subsystem
python_setup = self._python_setup_subsystem
# NB: self._all_find_links is updated on every call to self._resolve_multi(), and therefore
# includes all of the links from python_repos.repos, as well as any links added within any
# individual requirements from that resolve.
resolver_settings = dict(
indexes=list(python_repos.indexes),
find_links=list(self._all_find_links),
allow_prereleases=UnsetBool.coerce_bool(
python_setup.resolver_allow_prereleases, default=True),
manylinux=python_setup.manylinux,
)
# IPEX-INFO: A json mapping interpreted in ipex_launcher.py:
# {
# "code": [<which source files to add to the "hydrated" pex when bootstrapped>],
# "resolver_settings": {<which indices to search for requirements from when bootstrapping>},
# }
ipex_info = dict(
code=prefixed_code_paths,
resolver_settings=resolver_settings,
)
with temporary_file(permissions=0o644) as ipex_info_file:
ipex_info_file.write(json.dumps(ipex_info).encode())
ipex_info_file.flush()
self._builder.add_resource(filename=ipex_info_file.name,
env_filename="IPEX-INFO")
# BOOTSTRAP-PEX-INFO: The original PEX-INFO, which should be the PEX-INFO in the hydrated .pex
# file that is generated when the .ipex is first executed.
with temporary_file(permissions=0o644) as bootstrap_pex_info_file:
bootstrap_pex_info_file.write(orig_pex_info.dump().encode())
bootstrap_pex_info_file.flush()
self._builder.add_resource(filename=bootstrap_pex_info_file.name,
env_filename="BOOTSTRAP-PEX-INFO")
# ipex.py: The special bootstrap script to hydrate the .ipex with the fully resolved
# requirements when it is first executed.
# Extract the file contents of our custom app launcher script from the pants package.
parent_module = module_dirname(module_dirname(ipex_launcher.__name__))
ipex_launcher_provider = get_provider(parent_module)
ipex_launcher_script = ipex_launcher_provider.get_resource_string(
parent_module, "ipex/ipex_launcher.py")
with temporary_file(permissions=0o644) as ipex_launcher_file:
ipex_launcher_file.write(ipex_launcher_script)
ipex_launcher_file.flush()
# Our .ipex file will use our custom app launcher!
self._builder.set_executable(ipex_launcher_file.name,
env_filename="ipex.py")
# The PEX-INFO we generate shouldn't have any requirements (except pex itself), or they will
# fail to bootstrap because they were unable to find those distributions. Instead, the .pex file
# produced when the .ipex is first executed will read and resolve all those requirements from
# the BOOTSTRAP-PEX-INFO.
self.add_resolved_requirements(
[self._pex_requirement, self._setuptools_requirement],
override_ipex_build_do_actually_add_distribution=True,
)
def freeze(self) -> None:
if self._frozen:
return
if self._prepare_inits():
dist = self._distributions.get("setuptools")
if not dist:
self.add_resolved_requirements([self._setuptools_requirement])
if self._generate_ipex:
self._shuffle_original_build_info_into_ipex()
self._builder.freeze(bytecode_compile=False)
self._frozen = True
def set_entry_point(self, entry_point):
self._builder.set_entry_point(entry_point)
def build(self, safe_path):
self.freeze()
self._builder.build(safe_path,
bytecode_compile=False,
deterministic_timestamp=True)
def set_shebang(self, shebang):
self._builder.set_shebang(shebang)
def add_interpreter_constraint(self, constraint):
self._builder.add_interpreter_constraint(constraint)
def add_interpreter_constraints_from(self, constraint_tgts):
# TODO this would be a great place to validate the constraints and present a good error message
# if they are incompatible because all the sources of the constraints are available.
# See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py
constraint_tuples = {
self._python_setup_subsystem.compatibility_or_constraints(
tgt.compatibility)
for tgt in constraint_tgts
}
for constraint_tuple in constraint_tuples:
for constraint in constraint_tuple:
self.add_interpreter_constraint(constraint)
def add_direct_requirements(self, reqs):
for req in reqs:
self._builder.add_requirement(str(req))
def add_distribution(self, dist):
self._builder.add_distribution(dist)
self._register_distribution(dist)
def add_dist_location(self, location):
self._builder.add_dist_location(location)
dist = DistributionHelper.distribution_from_path(location)
self._register_distribution(dist)
def _register_distribution(self, dist):
self._distributions[dist.key] = dist
def set_script(self, script):
self._builder.set_script(script)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def build_pex(pex_filename: str, local_only: bool) -> str:
pex_builder = PEXBuilder(include_tools=True)
pex_builder.info.inherit_path = InheritPath.FALLBACK
pex_builder.set_entry_point('daml_dit_if.main:main')
pex_builder.set_shebang('/usr/bin/env python3')
platforms = [parsed_platform('current')]
if local_only:
LOG.warn('Local-only build. THIS DIT WILL NOT RUN IN DAML HUB.')
else:
platforms = [
*platforms,
parsed_platform('manylinux2014_x86_64-cp-38-cp38')
]
daml_dit_if_bundled = False
try:
if os.path.isfile(PYTHON_REQUIREMENT_FILE):
LOG.info(
f'Bundling dependencies from {PYTHON_REQUIREMENT_FILE}...')
requirement_files = [PYTHON_REQUIREMENT_FILE]
else:
LOG.info(
f'No dependency file found ({PYTHON_REQUIREMENT_FILE}), no dependencies will be bundled.'
)
requirement_files = []
resolveds = resolve_multi(requirements=[],
requirement_files=requirement_files,
platforms=platforms)
for resolved_dist in resolveds:
if resolved_dist.distribution.project_name == IF_PROJECT_NAME \
and not daml_dit_if_bundled:
LOG.warn(
f'Bundling {IF_PROJECT_NAME} in output DIT file. This will'
f' override the version provided by Daml Hub, potentially'
f' compromising compatibility of this integration with'
f' future updates to Daml Hub. Use this option with caution.'
)
daml_dit_if_bundled = True
LOG.debug("req: %s", resolved_dist.distribution)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
LOG.info("direct_req: %s", resolved_dist.direct_requirement)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_requirement(resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(f'Unsatifiable dependency error: {e}')
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
LOG.debug("Adding source file: %r, %r", src_file_path,
dst_path)
fn(src_file_path, dst_path)
walk_and_do(pex_builder.add_source, 'src/')
pex_builder.freeze(bytecode_compile=True)
# Entry point verification is disabled because ddit does not
# formally depend on the integration framework, and it is not
# necessarily available at integration build time. Because entry
# point verification happens in ddit's environment and the
# entrypoint is in the framework, this causes entry point
# verification to fail unless some other agent has installed
# daml-dit-if into ddit's environment.
#
# Virtual environments provide a way to work around this (and are
# used in 'ddit run') but the PEX API does not allow a virtual
# environment to be specified at build time. If this ever changes,
# the build subcommand should be modified to prepare a virtual
# enviroment for the build that contains the appropriate version
# of daml-dit-if and entrypoint verification should be re-enabled.
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=False)
LOG.info('Building intermediate PEX file...')
LOG.debug('PEX info: %r', pex_builder.info)
pex_builder.build(pex_filename,
bytecode_compile=True,
deterministic_timestamp=True)
if daml_dit_if_bundled:
return 'python-direct'
else:
return 'python-direct-hub-if'
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe',
action='store_false',
dest='zip_safe',
default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(options.python,
PythonInterpreter.from_binary(
options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
copy_package(pex_builder,
'pkg_resources',
prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on targets: %s' %
' '.join(str(target) for target in self.targets))
builder = PEXBuilder(
tempfile.mkdtemp(),
interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error(
'No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython',
'entry_point')
if builder.info.entry_point is None:
self.error(
'Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython',
'requirements',
default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(
self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def assert_force_local_implicit_ns_packages_issues_598(
interpreter=None, requirements=(), create_ns_packages=True
):
def create_foo_bar_setup(name, **extra_args):
# type: (str, **Any) -> str
setup_args = dict(name=name, version="0.0.1", packages=["foo", "foo.bar"])
if create_ns_packages:
setup_args.update(namespace_packages=["foo", "foo.bar"])
if requirements:
setup_args.update(install_requires=list(requirements))
setup_args.update(extra_args)
return dedent(
"""
from setuptools import setup
setup(**{setup_args!r})
""".format(
setup_args=setup_args
)
)
def with_foo_bar_ns_packages(content):
# type: (Dict[str, str]) -> Dict[str, str]
ns_packages = (
{
os.path.join(
pkg, "__init__.py"
): '__import__("pkg_resources").declare_namespace(__name__)'
for pkg in ("foo", "foo/bar")
}
if create_ns_packages
else {}
)
ns_packages.update(content)
return ns_packages
content1 = with_foo_bar_ns_packages(
{
"foo/bar/spam.py": "identify = lambda: 42",
"setup.py": create_foo_bar_setup("foo-bar-spam"),
}
)
content2 = with_foo_bar_ns_packages(
{
"foo/bar/eggs.py": dedent(
"""
# NB: This only works when this content is unpacked loose on the filesystem!
def read_self():
with open(__file__) as fp:
return fp.read()
"""
)
}
)
content3 = with_foo_bar_ns_packages(
{
"foobaz": dedent(
"""\
#!python
import sys
from foo.bar import baz
sys.exit(baz.main())
"""
),
"foo/bar/baz.py": dedent(
"""
import sys
from foo.bar import eggs, spam
def main():
assert len(eggs.read_self()) > 0
return spam.identify()
"""
),
"setup.py": create_foo_bar_setup("foo-bar-baz", scripts=["foobaz"]),
}
)
def add_requirements(builder, cache):
# type: (PEXBuilder, str) -> None
for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter):
builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
builder.add_requirement(resolved_dist.direct_requirement)
def add_wheel(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
dist = WheelBuilder(project, interpreter=builder.interpreter).bdist()
builder.add_dist_location(dist)
def add_sources(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
for path in content.keys():
builder.add_source(os.path.join(project, path), path)
with temporary_dir() as root, temporary_dir() as cache:
pex_info1 = PexInfo.default()
pex_info1.zip_safe = False
pex1 = os.path.join(root, "pex1.pex")
builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1)
add_requirements(builder1, cache)
add_wheel(builder1, content1)
add_sources(builder1, content2)
builder1.build(pex1)
pex_info2 = PexInfo.default()
pex_info2.pex_path = pex1
pex2 = os.path.join(root, "pex2")
builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2)
add_requirements(builder2, cache)
add_wheel(builder2, content3)
builder2.set_script("foobaz")
builder2.freeze()
assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE="9"))
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].items():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
