def _test_runner(self, targets, workunit):
interpreter = self.select_interpreter_for_targets(targets)
builder = PEXBuilder(interpreter=interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self._maybe_shard() as shard_args:
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
workunit) as coverage_args:
yield pex, shard_args + junit_args + coverage_args
finally:
chroot.delete()
def resolve_requirements(self, interpreter, req_libs):
"""Requirements resolution for PEX files.
:param interpreter: Resolve against this :class:`PythonInterpreter`.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:returns: a PEX containing target requirements and any specified python dist targets.
"""
with self.invalidated(req_libs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
tgts = self.context.targets()
maybe_platforms = ['current'] if build_for_current_platform_only_check(tgts) else None
path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id))
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True)
dump_requirement_libs(builder, interpreter, req_libs, self.context.log, platforms=maybe_platforms)
builder.freeze()
return PEX(path, interpreter=interpreter)
def _build_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
self.context.log.debug(" Dumping requirement: {}".format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
self.context.log.debug("Skipping {} based on version filter".format(req))
# Resolve the requirements into distributions.
distributions = self._resolve_multi(interpreter, reqs_to_build, find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self.context.log.debug(" Dumping distribution: .../{}".format(os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
builder.freeze()
def nsutil_pex(self):
interpreter = self.context.products.get_data(PythonInterpreter)
chroot = os.path.join(self.workdir, 'nsutil', interpreter.version_string)
if not os.path.exists(chroot):
pex_info = PexInfo.default(interpreter=interpreter)
with safe_concurrent_creation(chroot) as scratch:
builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True)
with temporary_file(binary_mode=False) as fp:
declares_namespace_package_code = inspect.getsource(declares_namespace_package)
fp.write(textwrap.dedent("""
import sys
{declares_namespace_package_code}
if __name__ == '__main__':
for path in sys.argv[1:]:
if declares_namespace_package(path):
print(path)
""").strip().format(declares_namespace_package_code=declares_namespace_package_code))
fp.close()
builder.set_executable(filename=fp.name, env_filename='main.py')
builder.freeze()
return PEX(pex=chroot, interpreter=interpreter)
def temporary_chroot(self, interpreter=None, pex_info=None, targets=None,
extra_requirements=None, platforms=None, pre_freeze=None):
"""Yields a temporary PythonChroot created with the specified args.
pre_freeze is an optional function run on the chroot just before freezing its builder,
to allow for any extra modification.
"""
path = tempfile.mkdtemp()
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
with self.context.new_workunit('chroot'):
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
if pre_freeze:
pre_freeze(chroot)
builder.freeze()
yield chroot
chroot.delete()
def write_pex(td, exe_contents):
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
reqs_pex_path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity),
vt.cache_key.hash))
if not os.path.isdir(reqs_pex_path):
req_libs = filter(has_python_requirements, vt.target.closure())
with safe_concurrent_creation(reqs_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True)
dump_requirement_libs(builder, interpreter, req_libs, self.context.log)
builder.freeze()
return PEX(reqs_pex_path, interpreter=interpreter)
def create_pex(self, pex_info=None):
"""Returns a wrapped pex that "merges" the other pexes via PEX_PATH."""
relevant_targets = self.context.targets(
lambda tgt: isinstance(tgt, (PythonRequirementLibrary, PythonTarget, Resources)))
with self.invalidated(relevant_targets) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter = self.context.products.get_data(PythonInterpreter)
path = os.path.join(self.workdir, str(interpreter.identity), target_set_id)
extra_pex_paths_file_path = path + '.extra_pex_paths'
extra_pex_paths = None
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
pexes = [
self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(GatherSources.PYTHON_SOURCES)
]
if self.extra_requirements():
extra_reqs = [PythonRequirement(req_str) for req_str in self.extra_requirements()]
addr = Address.parse('{}_extra_reqs'.format(self.__class__.__name__))
self.context.build_graph.inject_synthetic_target(
addr, PythonRequirementLibrary, requirements=extra_reqs)
# Add the extra requirements first, so they take precedence over any colliding version
# in the target set's dependency closure.
pexes = [self.resolve_requirements([self.context.build_graph.get_target(addr)])] + pexes
extra_pex_paths = [pex.path() for pex in pexes if pex]
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
builder.freeze()
with open(extra_pex_paths_file_path, 'w') as outfile:
for epp in extra_pex_paths:
outfile.write(epp)
outfile.write(b'\n')
if extra_pex_paths is None:
with open(extra_pex_paths_file_path, 'r') as infile:
extra_pex_paths = [p.strip() for p in infile.readlines()]
return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter)
def merge_pexes(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None):
"""Generates a merged pex at path."""
pex_paths = [pex.path() for pex in pexes if pex]
if pex_paths:
pex_info = pex_info.copy()
pex_info.merge_pex_path(':'.join(pex_paths))
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
if interpeter_constraints:
for constraint in interpeter_constraints:
builder.add_interpreter_constraint(constraint)
builder.freeze()
def write_pex(td, exe_contents, dists=None):
dists = dists or []
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
for dist in dists:
pb.add_egg(dist.location)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def bootstrap_conan(self):
pex_info = PexInfo.default()
pex_info.entry_point = 'conans.conan'
conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
interpreter = PythonInterpreter.get()
if not os.path.exists(conan_pex_path):
with safe_concurrent_creation(conan_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
dump_requirements(builder, interpreter, reqs, logger)
builder.freeze()
conan_binary = PEX(conan_pex_path, interpreter)
return self.ConanBinary(pex=conan_binary)
def write_simple_pex(td,
exe_contents=None,
dists=None,
sources=None,
coverage=False,
interpreter=None,
pex_info=None):
"""Write a pex file that optionally contains an executable entry point.
:param str td: temporary directory path
:param str exe_contents: entry point python file
:param dists: distributions to include, typically sdists or bdists
:type: list of :class:`pex.third_party.pkg_resources.Distribution`
:param sources: sources to include, as a list of pairs (env_filename, contents)
:type sources: list of (str, str)
:param bool coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:param pex_info: a custom PexInfo to use to build the pex.
:type pex_info: :class:`pex.pex_info.PexInfo`
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter,
pex_info=pex_info)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
if exe_contents:
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def test_execute_interpreter_dashc_program():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.freeze()
process = PEX(pex_chroot).run(
args=["-c", 'import sys; print(" ".join(sys.argv))', "one"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False,
)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b"-c one\n" == stdout
assert b"" == stderr
def _resolve_requirements_for_versioned_target_closure(
self, interpreter, vt):
reqs_pex_path = os.path.realpath(
os.path.join(self.workdir, str(interpreter.identity),
vt.cache_key.hash))
if not os.path.isdir(reqs_pex_path):
req_libs = filter(has_python_requirements, vt.target.closure())
with safe_concurrent_creation(reqs_pex_path) as safe_path:
builder = PEXBuilder(safe_path,
interpreter=interpreter,
copy=True)
dump_requirements(builder, interpreter, req_libs,
self.context.log)
builder.freeze()
return PEX(reqs_pex_path, interpreter=interpreter)
def _add_test_hello_to_pex(ep):
with temporary_dir() as td:
hello_file = "\n".join([
"def hello():",
" print('hello')",
])
with temporary_file(root_dir=td) as tf:
with open(tf.name, 'w') as handle:
handle.write(hello_file)
pex_builder = PEXBuilder()
pex_builder.add_source(tf.name, 'test.py')
pex_builder.set_entry_point(ep)
pex_builder.freeze()
yield pex_builder
def test_execute_interpreter_stdin_program():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.freeze()
process = PEX(pex_chroot).run(args=['-', 'one', 'two'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate(
input=b'import sys; print(" ".join(sys.argv))')
assert 0 == process.returncode
assert b'- one two\n' == stdout
assert b'' == stderr
def test_pex_executable():
# Tests that pex keeps executable permissions
with temporary_dir() as temp_dir:
pex_dir = os.path.join(temp_dir, 'pex_dir')
safe_mkdir(pex_dir)
with open(os.path.join(pex_dir, 'exe.py'), 'w') as fp:
fp.write(textwrap.dedent('''
import subprocess
import os
import sys
import my_package
path = os.path.join(os.path.dirname(my_package.__file__), 'bin/start.sh')
sys.stdout.write(subprocess.check_output([path]).decode('utf-8'))
'''))
project_content = {
'setup.py': textwrap.dedent('''
from setuptools import setup
setup(
name='my_project',
version='0.0.0.0',
zip_safe=True,
packages=['my_package'],
package_data={'my_package': ['bin/*']},
install_requires=[],
)
'''),
'my_package/__init__.py': 0,
'my_package/bin/start.sh': ("#!/usr/bin/env bash\n"
"echo 'hello world from start.sh!'"),
'my_package/my_module.py': 'def do_something():\n print("hello world!")\n',
}
pex_builder = PEXBuilder(path=pex_dir)
with temporary_content(project_content, perms=0o755) as project_dir:
installer = WheelBuilder(project_dir)
bdist = installer.bdist()
pex_builder.add_dist_location(bdist)
pex_builder.set_executable(os.path.join(pex_dir, 'exe.py'))
pex_builder.freeze()
app_pex = os.path.join(os.path.join(temp_dir, 'out_pex_dir'), 'app.pex')
pex_builder.build(app_pex)
std_out, rc = run_simple_pex(app_pex,
env={'PEX_ROOT': os.path.join(temp_dir, '.pex')})
assert rc == 0
assert std_out.decode('utf-8') == 'hello world from start.sh!\n'
def test_execute_interpreter_file_program():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.freeze()
with tempfile.NamedTemporaryFile() as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.flush()
process = PEX(pex_chroot).run(args=[fp.name, 'one', 'two'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert '{} one two\n'.format(fp.name).encode('utf-8') == stdout
assert b'' == stderr
def test_execute_interpreter_dashm_module():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
with temporary_file(root_dir=pex_chroot) as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.close()
pex_builder.add_source(fp.name, 'foo/bar.py')
pex_builder.freeze()
process = PEX(pex_chroot).run(args=['-m', 'foo.bar', 'one', 'two'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b'foo.bar one two\n' == stdout
assert b'' == stderr
def test_execute_interpreter_stdin_program():
# type: () -> None
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.freeze()
process = PEX(pex_chroot).run(
args=["-", "one", "two"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
blocking=False,
)
stdout, stderr = process.communicate(input=b'import sys; print(" ".join(sys.argv))')
assert 0 == process.returncode
assert b"- one two\n" == stdout
assert b"" == stderr
def write_and_run_simple_pex(inheriting=False):
"""Write a pex file that contains an executable entry point
:param inheriting: whether this pex should inherit site-packages paths
:type inheriting: bool
"""
with temporary_dir() as td:
pex_path = os.path.join(td, 'show_path.pex')
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write('') # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
pb.build(pex_path)
yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0]
def bootstrap_conan(self):
pex_info = PexInfo.default()
pex_info.entry_point = 'conans.conan'
conan_bootstrap_dir = os.path.join(get_pants_cachedir(),
'conan_support')
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
interpreter = PythonInterpreter.get()
if not os.path.exists(conan_pex_path):
with safe_concurrent_creation(conan_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
reqs = [
PythonRequirement(req)
for req in self.get_options().conan_requirements
]
dump_requirements(builder, interpreter, reqs, logger)
builder.freeze()
conan_binary = PEX(conan_pex_path, interpreter)
return self.ConanBinary(pex=conan_binary)
def resolve_requirement_strings(self, interpreter, requirement_strings):
"""Resolve a list of pip-style requirement strings."""
requirement_strings = sorted(requirement_strings)
if len(requirement_strings) == 0:
req_strings_id = 'no_requirements'
elif len(requirement_strings) == 1:
req_strings_id = requirement_strings[0]
else:
req_strings_id = hash_all(requirement_strings)
path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), req_strings_id))
if not os.path.isdir(path):
reqs = [PythonRequirement(req_str) for req_str in requirement_strings]
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True)
dump_requirements(builder, interpreter, reqs, self.context.log)
builder.freeze()
return PEX(path, interpreter=interpreter)
def write_and_run_simple_pex(inheriting):
# type: (str) -> Iterator[Text]
"""Write a pex file that contains an executable entry point.
:param inheriting: whether this pex should inherit site-packages paths.
"""
with temporary_dir() as td:
pex_path = os.path.join(td, "show_path.pex")
with open(os.path.join(td, "exe.py"), "w") as fp:
fp.write(u"") # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, "exe.py"))
pb.freeze()
pb.build(pex_path)
stdout, _ = run_simple_pex(pex_path, env={"PEX_VERBOSE": "1"})
yield stdout.decode("utf-8")
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Use MutexTask to coordinate this.
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create a wrapper pex to "merge" the other pexes into via PEX_PATH.
builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo)
builder.freeze()
pexes = [
self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(GatherSources.PYTHON_SOURCES),
]
# TODO: Expose the path as a property in pex, instead of relying on
# fishing it out of the cmdline.
pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes])
pex = PEX(tmpdir, interpreter)
self.context.release_lock()
with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]):
args = []
for arg in self.get_options().args:
args.extend(safe_shlex_split(arg))
args += self.get_passthru_args()
po = pex.run(blocking=False, args=args, env={"PEX_PATH": pex_path})
try:
result = po.wait()
if result != 0:
msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format(
interpreter=interpreter.binary,
entry_point=binary.entry_point,
args=" ".join(args),
code=result,
)
raise TaskError(msg, exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def build_and_check(path, precompile):
pb = PEXBuilder(path)
pb.add_source(src, 'lib/src.py')
pb.set_executable(exe, 'exe.py')
pb.freeze(bytecode_compile=precompile)
for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
bootstrap_pycs = []
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
def resolve_requirements(self, interpreter, req_libs):
"""Requirements resolution for PEX files.
:param interpreter: Resolve against this :class:`PythonInterpreter`.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:returns: a PEX containing target requirements and any specified python dist targets.
"""
with self.invalidated(req_libs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
tgts = self.context.targets()
if self._python_native_code_settings.check_build_for_current_platform_only(
tgts):
maybe_platforms = ['current']
else:
maybe_platforms = None
path = os.path.realpath(
os.path.join(self.workdir, str(interpreter.identity),
target_set_id))
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(path=safe_path,
interpreter=interpreter,
copy=True)
dump_requirement_libs(builder,
interpreter,
req_libs,
self.context.log,
platforms=maybe_platforms)
builder.freeze()
return PEX(path, interpreter=interpreter)
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(
itertools.chain(
*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources,
blocking=True,
setsid=True,
stdout=stdout,
stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets, builder.path(),
pex, stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def write_simple_pex(td,
exe_contents,
dists=None,
sources=None,
coverage=False,
interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def test_activate_interpreter_different_from_current():
with temporary_dir() as pex_root:
interp_version = PY36 if PY2 else PY27
custom_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(interp_version))
pex_info = PexInfo.default(custom_interpreter)
pex_info.pex_root = pex_root
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot,
interpreter=custom_interpreter,
pex_info=pex_info)
with make_bdist(interpreter=custom_interpreter) as bdist:
pex_builder.add_distribution(bdist)
pex_builder.set_entry_point('sys:exit')
pex_builder.freeze()
pex = PEX(pex_builder.path(), interpreter=custom_interpreter)
try:
pex._activate()
except SystemExit as e:
pytest.fail('PEX activation of %s failed with %s' % (pex, e))
def build_and_check(path, precompile):
# type: (str, bool) -> None
pb = PEXBuilder(path=path)
pb.add_source(src, "lib/src.py")
pb.set_executable(exe, "exe.py")
pb.freeze(bytecode_compile=precompile)
for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, pb.info.bootstrap)
bootstrap_pycs = [] # type: List[str]
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith(".pyc"))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
def create(cls, path):
# type: (str) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to build the pip tool pex at.
"""
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if chroot is not None:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot)
pythonpath = third_party.expose(["pip", "setuptools", "wheel"])
isolated_pip_environment = third_party.pkg_resources.Environment(
search_path=pythonpath
)
for dist_name in isolated_pip_environment:
for dist in isolated_pip_environment[dist_name]:
isolated_pip_builder.add_dist_location(dist=dist.location)
with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp:
fp.write(
dedent(
"""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to
# perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""
)
)
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(pip_pex_path)
def _build_chroot(self, path, interpreter, pex_info, targets, platforms,
extra_requirements=None, executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info, copy=True)
with self.context.new_workunit('chroot'):
chroot = self.create_chroot(
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
def assert_namespace_packages_warning(distribution, version, expected_warning):
# type: (str, str, bool) -> None
requirement = "{}=={}".format(distribution, version)
pb = PEXBuilder()
for resolved_dist in resolver.resolve([requirement]):
pb.add_dist_location(resolved_dist.distribution.location)
pb.freeze()
process = PEX(pb.path()).run(args=["-c", ""], blocking=False, stderr=subprocess.PIPE)
_, stderr = process.communicate()
stderr_text = stderr.decode("utf8")
partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded"
partial_warning_detail = "{} namespace packages:".format(requirement)
if expected_warning:
assert partial_warning_preamble in stderr_text
assert partial_warning_detail in stderr_text
else:
assert partial_warning_preamble not in stderr_text
assert partial_warning_detail not in stderr_text
def test_execute_interpreter_dashm_module():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.add_source(None, "foo/__init__.py")
with tempfile.NamedTemporaryFile() as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.flush()
pex_builder.add_source(fp.name, "foo/bar.py")
pex_builder.freeze()
pex = PEX(pex_chroot)
process = pex.run(
args=["-m", "foo.bar", "one", "two"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False,
)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b"foo.bar one two\n" == stdout
assert b"" == stderr
def _build_chroot(self, path, interpreter, pex_info, targets, platforms,
extra_requirements=None, executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info, copy=True)
with self.context.new_workunit('chroot'):
chroot = self.create_chroot(
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def _build_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
self.context.log.debug(' Dumping requirement: {}'.format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
self.context.log.debug(
'Skipping {} based on version filter'.format(req))
# Resolve the requirements into distributions.
distributions = self._resolve_multi(interpreter, reqs_to_build,
find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self.context.log.debug(
' Dumping distribution: .../{}'.format(
os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
builder.freeze()
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def create_pex_repository(
interpreters=None, # type: Optional[Iterable[PythonInterpreter]]
platforms=None, # type: Optional[Iterable[Platform]]
requirements=None, # type: Optional[Iterable[str]]
requirement_files=None, # type: Optional[Iterable[str]]
constraint_files=None, # type: Optional[Iterable[str]]
manylinux=None, # type: Optional[str]
):
# type: (...) -> str
pex_builder = PEXBuilder()
for resolved_dist in resolve_multi(
interpreters=interpreters,
platforms=platforms,
requirements=requirements,
requirement_files=requirement_files,
constraint_files=constraint_files,
manylinux=manylinux,
):
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(resolved_dist.direct_requirement)
pex_builder.freeze()
return os.path.realpath(cast(str, pex_builder.path()))
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def test_pex_run_strip_env():
with temporary_dir() as pex_root:
pex_env = dict(PEX_MODULE='does_not_exist_in_sub_pex', PEX_ROOT=pex_root)
with environment_as(**pex_env), temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
with tempfile.NamedTemporaryFile(mode="w") as fp:
fp.write(dedent("""
import json
import os
print(json.dumps({k: v for k, v in os.environ.items() if k.startswith("PEX_")}))
"""))
fp.flush()
pex_builder.set_executable(fp.name, 'print_pex_env.py')
pex_builder.freeze()
stdout, returncode = run_simple_pex(pex_chroot)
assert 0 == returncode
assert {} == json.loads(stdout.decode('utf-8')), (
'Expected the entrypoint environment to be stripped of PEX_ environment variables.'
)
assert pex_env == {k: v for k, v in os.environ.items() if k.startswith("PEX_")}, (
'Expected the parent environment to be left un-stripped.'
)
def _build_pex(self, interpreter, path, vts):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
for vt in vts:
self._dump_sources(builder, vt.target)
builder.freeze()
def _build_pex(self, interpreter, path, targets):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
for target in targets:
dump_sources(builder, target, self.context.log)
builder.freeze()
def _build_requirements_pex(self, interpreter, path, req_libs): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) dump_requirements(builder, interpreter, req_libs, self.context.log) builder.freeze()
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(reqs_pex.path().encode('utf-8'))
hasher.update(srcs_pex.path().encode('utf-8'))
hasher.update(executable_file_content.encode('utf-8'))
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(os.path.join(safe_path, '{}.py'.format(self._EXEC_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
pex_info.pex_path = ':'.join(pex.path() for pex in (reqs_pex, srcs_pex) if pex)
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
builder.freeze()
pex = PEX(exec_pex_path, interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
def _build_requirements_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
dump_requirements(builder, interpreter, req_libs, self.context.log)
builder.freeze()
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def assert_force_local_implicit_ns_packages_issues_598(
interpreter=None, requirements=(), create_ns_packages=True
):
def create_foo_bar_setup(name, **extra_args):
# type: (str, **Any) -> str
setup_args = dict(name=name, version="0.0.1", packages=["foo", "foo.bar"])
if create_ns_packages:
setup_args.update(namespace_packages=["foo", "foo.bar"])
if requirements:
setup_args.update(install_requires=list(requirements))
setup_args.update(extra_args)
return dedent(
"""
from setuptools import setup
setup(**{setup_args!r})
""".format(
setup_args=setup_args
)
)
def with_foo_bar_ns_packages(content):
# type: (Dict[str, str]) -> Dict[str, str]
ns_packages = (
{
os.path.join(
pkg, "__init__.py"
): '__import__("pkg_resources").declare_namespace(__name__)'
for pkg in ("foo", "foo/bar")
}
if create_ns_packages
else {}
)
ns_packages.update(content)
return ns_packages
content1 = with_foo_bar_ns_packages(
{
"foo/bar/spam.py": "identify = lambda: 42",
"setup.py": create_foo_bar_setup("foo-bar-spam"),
}
)
content2 = with_foo_bar_ns_packages(
{
"foo/bar/eggs.py": dedent(
"""
# NB: This only works when this content is unpacked loose on the filesystem!
def read_self():
with open(__file__) as fp:
return fp.read()
"""
)
}
)
content3 = with_foo_bar_ns_packages(
{
"foobaz": dedent(
"""\
#!python
import sys
from foo.bar import baz
sys.exit(baz.main())
"""
),
"foo/bar/baz.py": dedent(
"""
import sys
from foo.bar import eggs, spam
def main():
assert len(eggs.read_self()) > 0
return spam.identify()
"""
),
"setup.py": create_foo_bar_setup("foo-bar-baz", scripts=["foobaz"]),
}
)
def add_requirements(builder, cache):
# type: (PEXBuilder, str) -> None
for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter):
builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
builder.add_requirement(resolved_dist.direct_requirement)
def add_wheel(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
dist = WheelBuilder(project, interpreter=builder.interpreter).bdist()
builder.add_dist_location(dist)
def add_sources(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
for path in content.keys():
builder.add_source(os.path.join(project, path), path)
with temporary_dir() as root, temporary_dir() as cache:
pex_info1 = PexInfo.default()
pex_info1.zip_safe = False
pex1 = os.path.join(root, "pex1.pex")
builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1)
add_requirements(builder1, cache)
add_wheel(builder1, content1)
add_sources(builder1, content2)
builder1.build(pex1)
pex_info2 = PexInfo.default()
pex_info2.pex_path = pex1
pex2 = os.path.join(root, "pex2")
builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2)
add_requirements(builder2, cache)
add_wheel(builder2, content3)
builder2.set_script("foobaz")
builder2.freeze()
assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE="9"))
