def test_pex_builder_deterministic_timestamp():
pb = PEXBuilder()
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target, deterministic_timestamp=True)
with zipfile.ZipFile(target) as zf:
assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())
def _test_runner(self, targets, workunit):
interpreter = self.select_interpreter_for_targets(targets)
builder = PEXBuilder(interpreter=interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self._maybe_shard() as shard_args:
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
workunit) as coverage_args:
yield pex, shard_args + junit_args + coverage_args
finally:
chroot.delete()
def nsutil_pex(self):
interpreter = self.context.products.get_data(PythonInterpreter)
chroot = os.path.join(self.workdir, 'nsutil', interpreter.version_string)
if not os.path.exists(chroot):
pex_info = PexInfo.default(interpreter=interpreter)
with safe_concurrent_creation(chroot) as scratch:
builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True)
with temporary_file(binary_mode=False) as fp:
declares_namespace_package_code = inspect.getsource(declares_namespace_package)
fp.write(textwrap.dedent("""
import sys
{declares_namespace_package_code}
if __name__ == '__main__':
for path in sys.argv[1:]:
if declares_namespace_package(path):
print(path)
""").strip().format(declares_namespace_package_code=declares_namespace_package_code))
fp.close()
builder.set_executable(filename=fp.name, env_filename='main.py')
builder.freeze()
return PEX(pex=chroot, interpreter=interpreter)
def temporary_chroot(self, interpreter=None, pex_info=None, targets=None,
extra_requirements=None, platforms=None, pre_freeze=None):
"""Yields a temporary PythonChroot created with the specified args.
pre_freeze is an optional function run on the chroot just before freezing its builder,
to allow for any extra modification.
"""
path = tempfile.mkdtemp()
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
with self.context.new_workunit('chroot'):
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
if pre_freeze:
pre_freeze(chroot)
builder.freeze()
yield chroot
chroot.delete()
def resolve_requirements(self, interpreter, req_libs):
"""Requirements resolution for PEX files.
:param interpreter: Resolve against this :class:`PythonInterpreter`.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:returns: a PEX containing target requirements and any specified python dist targets.
"""
with self.invalidated(req_libs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
tgts = self.context.targets()
maybe_platforms = ['current'] if build_for_current_platform_only_check(tgts) else None
path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id))
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True)
dump_requirement_libs(builder, interpreter, req_libs, self.context.log, platforms=maybe_platforms)
builder.freeze()
return PEX(path, interpreter=interpreter)
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def write_pex(td, exe_contents):
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def test_pex_builder_shebang():
pb = PEXBuilder()
pb.set_shebang('foobar')
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
pb.build(target)
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
reqs_pex_path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity),
vt.cache_key.hash))
if not os.path.isdir(reqs_pex_path):
req_libs = filter(has_python_requirements, vt.target.closure())
with safe_concurrent_creation(reqs_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True)
dump_requirement_libs(builder, interpreter, req_libs, self.context.log)
builder.freeze()
return PEX(reqs_pex_path, interpreter=interpreter)
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
s1 = os.stat(src)
s2 = os.stat(os.path.join(path, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def create_pex(self, pex_info=None):
"""Returns a wrapped pex that "merges" the other pexes via PEX_PATH."""
relevant_targets = self.context.targets(
lambda tgt: isinstance(tgt, (PythonRequirementLibrary, PythonTarget, Resources)))
with self.invalidated(relevant_targets) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter = self.context.products.get_data(PythonInterpreter)
path = os.path.join(self.workdir, str(interpreter.identity), target_set_id)
extra_pex_paths_file_path = path + '.extra_pex_paths'
extra_pex_paths = None
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
pexes = [
self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(GatherSources.PYTHON_SOURCES)
]
if self.extra_requirements():
extra_reqs = [PythonRequirement(req_str) for req_str in self.extra_requirements()]
addr = Address.parse('{}_extra_reqs'.format(self.__class__.__name__))
self.context.build_graph.inject_synthetic_target(
addr, PythonRequirementLibrary, requirements=extra_reqs)
# Add the extra requirements first, so they take precedence over any colliding version
# in the target set's dependency closure.
pexes = [self.resolve_requirements([self.context.build_graph.get_target(addr)])] + pexes
extra_pex_paths = [pex.path() for pex in pexes if pex]
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
builder.freeze()
with open(extra_pex_paths_file_path, 'w') as outfile:
for epp in extra_pex_paths:
outfile.write(epp)
outfile.write(b'\n')
if extra_pex_paths is None:
with open(extra_pex_paths_file_path, 'r') as infile:
extra_pex_paths = [p.strip() for p in infile.readlines()]
return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter)
def merge_pexes(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None):
"""Generates a merged pex at path."""
pex_paths = [pex.path() for pex in pexes if pex]
if pex_paths:
pex_info = pex_info.copy()
pex_info.merge_pex_path(':'.join(pex_paths))
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
if interpeter_constraints:
for constraint in interpeter_constraints:
builder.add_interpreter_constraint(constraint)
builder.freeze()
def bootstrap_conan(self):
pex_info = PexInfo.default()
pex_info.entry_point = 'conans.conan'
conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
interpreter = PythonInterpreter.get()
if not os.path.exists(conan_pex_path):
with safe_concurrent_creation(conan_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
dump_requirements(builder, interpreter, reqs, logger)
builder.freeze()
conan_binary = PEX(conan_pex_path, interpreter)
return self.ConanBinary(pex=conan_binary)
def build_and_check(path, copy):
pb = PEXBuilder(path, copy=copy)
pb.add_source(src, 'exe.py')
path_clone = os.path.join(path, '__clone')
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, 'exe.py'))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def merged_pex(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None):
"""Yields a pex builder at path with the given pexes already merged.
:rtype: :class:`pex.pex_builder.PEXBuilder`
"""
pex_paths = [pex.path() for pex in pexes if pex]
if pex_paths:
pex_info = pex_info.copy()
pex_info.merge_pex_path(':'.join(pex_paths))
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
if interpeter_constraints:
for constraint in interpeter_constraints:
builder.add_interpreter_constraint(constraint)
yield builder
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _build(self):
target_path = os.path.join(self.install_directory,
self.target.get("executable_name"))
parser, resolver_options_builder = configure_clp()
pex_builder = PEXBuilder(
interpreter=self._get_interpreter(parser)
)
pex_builder.set_entry_point(self.target.get("entry_point"))
for dist in self._get_distributions(resolver_options_builder):
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.build(target_path)
self.directory.symlink_to_bin(self.target.get("executable_name"),
target_path)
def _build_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
self.context.log.debug(" Dumping requirement: {}".format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
self.context.log.debug("Skipping {} based on version filter".format(req))
# Resolve the requirements into distributions.
distributions = self._resolve_multi(interpreter, reqs_to_build, find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self.context.log.debug(" Dumping distribution: .../{}".format(os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
builder.freeze()
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def resolve_requirement_strings(self, interpreter, requirement_strings):
"""Resolve a list of pip-style requirement strings."""
requirement_strings = sorted(requirement_strings)
if len(requirement_strings) == 0:
req_strings_id = 'no_requirements'
elif len(requirement_strings) == 1:
req_strings_id = requirement_strings[0]
else:
req_strings_id = hash_all(requirement_strings)
path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), req_strings_id))
if not os.path.isdir(path):
reqs = [PythonRequirement(req_str) for req_str in requirement_strings]
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True)
dump_requirements(builder, interpreter, reqs, self.context.log)
builder.freeze()
return PEX(path, interpreter=interpreter)
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Use MutexTask to coordinate this.
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create a wrapper pex to "merge" the other pexes into via PEX_PATH.
builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo)
builder.freeze()
pexes = [
self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(GatherSources.PYTHON_SOURCES),
]
# TODO: Expose the path as a property in pex, instead of relying on
# fishing it out of the cmdline.
pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes])
pex = PEX(tmpdir, interpreter)
self.context.release_lock()
with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]):
args = []
for arg in self.get_options().args:
args.extend(safe_shlex_split(arg))
args += self.get_passthru_args()
po = pex.run(blocking=False, args=args, env={"PEX_PATH": pex_path})
try:
result = po.wait()
if result != 0:
msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format(
interpreter=interpreter.binary,
entry_point=binary.entry_point,
args=" ".join(args),
code=result,
)
raise TaskError(msg, exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def build_pex(args):
with TRACER.timed('Resolving interpreter', V=2):
interpreter = _establish_interpreter(args)
if interpreter is None:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE)
pex_info = pex_builder.info
pex_info.zip_safe = False
pex_info.always_write_cache = True
pex_info.inherit_path = False
resolver_option_builder = _establish_resolver_options(args)
reqs = args.reqs
resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs]
for requirements_txt in args.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
resolver_kwargs = dict(interpreter=interpreter, platform=args.platform)
_add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder)
if not args.disable_cache:
resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs)
else:
resolver = Resolver(**resolver_kwargs)
resolveds = []
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolver.resolve(resolvables)
except Unsatisfiable as exception:
die(exception)
for dist in resolveds:
log(' %s' % dist, verbose=args.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.set_entry_point('spex:spex')
if args.python_shebang:
pex_builder.set_shebang(args.python_shebang)
return pex_builder
def _build_chroot(self, path, interpreter, pex_info, targets, platforms,
extra_requirements=None, executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info, copy=True)
with self.context.new_workunit('chroot'):
chroot = self.create_chroot(
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def test_pex_builder_preamble():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
should_create = os.path.join(td, 'foo.1')
tempfile_preamble = "\n".join([
"import sys",
"open('{0}', 'w').close()".format(should_create),
"sys.exit(3)"
])
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def write_pex(td, exe_contents, dists=None):
dists = dists or []
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
for dist in dists:
pb.add_egg(dist.location)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
if binary_tgt.shebang:
self.context.log.info('Found Python binary target {} with customized shebang, using it: {}'
.format(binary_tgt.name, binary_tgt.shebang))
builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug('No customized shebang found for {}'.format(binary_tgt.name))
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Add target's interpreter compatibility constraints to pex info.
if is_python_target(tgt):
for constraint in tgt.compatibility:
builder.add_interpreter_constraint(constraint)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
self._python_native_code_settings.check_build_for_current_platform_only(self.context.targets())
dump_requirement_libs(builder, interpreter, req_tgts, self.context.log,
platforms=binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def write_and_run_simple_pex(inheriting=False):
"""Write a pex file that contains an executable entry point
:param inheriting: whether this pex should inherit site-packages paths
:type inheriting: bool
"""
with temporary_dir() as td:
pex_path = os.path.join(td, 'show_path.pex')
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write('') # No contents, we just want the startup messages
pb = PEXBuilder(path=td, preamble=None)
pb.info.inherit_path = inheriting
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
pb.build(pex_path)
yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0]
def build_and_check(path, copy):
pb = PEXBuilder(path=path, copy=copy)
pb.add_source(src, 'exe.py')
path_clone = os.path.join(path, '__clone')
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, 'exe.py'))
is_link = (s1[stat.ST_INO],
s1[stat.ST_DEV]) == (s2[stat.ST_INO],
s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
po = subprocess.Popen([pex],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
po.wait()
output = po.stdout.read()
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert po.returncode == 0
def test_venv_symlinked_source_issues_1239(tmpdir):
# type: (Any) -> None
src = os.path.join(str(tmpdir), "src")
main = os.path.join(src, "main.py")
with safe_open(main, "w") as fp:
fp.write("import sys; sys.exit(42)")
pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK)
pex_builder.set_executable(main)
pex_file = os.path.join(str(tmpdir), "a.pex")
pex_builder.build(pex_file, bytecode_compile=False)
assert 42 == subprocess.Popen(args=[pex_file]).wait()
venv = os.path.join(str(tmpdir), "a.venv")
subprocess.check_call(args=[
sys.executable, "-m", "pex.tools",
pex_builder.path(), "venv", venv
])
venv_pex = os.path.join(venv, "pex")
shutil.rmtree(src)
assert 42 == subprocess.Popen(args=[venv_pex]).wait()
def assert_namespace_packages_warning(distribution, version, expected_warning):
# type: (str, str, bool) -> None
requirement = "{}=={}".format(distribution, version)
pb = PEXBuilder()
for resolved_dist in resolver.resolve([requirement]):
pb.add_dist_location(resolved_dist.distribution.location)
pb.freeze()
process = PEX(pb.path()).run(args=["-c", ""], blocking=False, stderr=subprocess.PIPE)
_, stderr = process.communicate()
stderr_text = stderr.decode("utf8")
partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded"
partial_warning_detail = "{} namespace packages:".format(requirement)
if expected_warning:
assert partial_warning_preamble in stderr_text
assert partial_warning_detail in stderr_text
else:
assert partial_warning_preamble not in stderr_text
assert partial_warning_detail not in stderr_text
def __init__(self,
context,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
self._config = Config.from_cache()
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def build_and_check(path, copy):
# type: (str, bool) -> None
pb = PEXBuilder(path=path, copy=copy)
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO],
s1[stat.ST_DEV]) == (s2[stat.ST_INO],
s2[stat.ST_DEV])
if copy:
assert not is_link
else:
assert is_link
def build_and_check(copy_mode):
# type: (CopyMode.Value) -> None
pb = PEXBuilder(copy_mode=copy_mode)
path = pb.path()
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy_mode == CopyMode.COPY:
assert not is_link
else:
# Since os.stat follows symlinks; so in CopyMode.SYMLINK, this just proves
# the symlink points to the original file. Going further and checking path
# and path_clone for the presence of a symlink (an os.islink test) is
# trickier since a Linux hardlink of a symlink produces a symlink whereas a
# macOS hardlink of a symlink produces a hardlink.
assert is_link
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets, builder.path(),
pex, stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties = PexInfo.make_build_properties()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
if binary_tgt.shebang:
self.context.log.info('Found Python binary target {} with customized shebang, using it: {}'
.format(binary_tgt.name, binary_tgt.shebang))
builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug('No customized shebang found for {}'.format(binary_tgt.name))
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
# Dump everything into the builder's chroot.
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
builder.build(pex_path)
return pex_path
def test_pex_run_strip_env():
with temporary_dir() as pex_root:
pex_env = dict(PEX_MODULE='does_not_exist_in_sub_pex', PEX_ROOT=pex_root)
with environment_as(**pex_env), temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
with tempfile.NamedTemporaryFile(mode="w") as fp:
fp.write(dedent("""
import json
import os
print(json.dumps({k: v for k, v in os.environ.items() if k.startswith("PEX_")}))
"""))
fp.flush()
pex_builder.set_executable(fp.name, 'print_pex_env.py')
pex_builder.freeze()
stdout, returncode = run_simple_pex(pex_chroot)
assert 0 == returncode
assert {} == json.loads(stdout.decode('utf-8')), (
'Expected the entrypoint environment to be stripped of PEX_ environment variables.'
)
assert pex_env == {k: v for k, v in os.environ.items() if k.startswith("PEX_")}, (
'Expected the parent environment to be left un-stripped.'
)
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(
interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(
interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(
exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(executable_file_content)
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(
os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(
os.path.join(safe_path,
'{}.py'.format(self._EXEC_NAME)),
'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(
target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
builder = PEXBuilder(safe_path,
interpreter,
pex_info=pex_info)
builder.freeze()
exec_pex = PEX(exec_pex_path, interpreter)
extra_pex_paths = [
pex.path()
for pex in [_f for _f in [reqs_pex, srcs_pex] if _f]
]
pex = WrappedPEX(exec_pex, extra_pex_paths)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL
],
cmd=' '.join(exec_pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
interpreters = [
get_interpreter(interpreter, options.interpreter_cache_dir,
options.repos, options.use_wheel)
for interpreter in options.python or [None]
]
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = _lowest_version_interpreter(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
resolvables = [
Resolvable.get(arg, resolver_option_builder) for arg in args
]
for requirements_txt in options.requirement_files:
resolvables.extend(
requirements_from_file(requirements_txt, resolver_option_builder))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
resolver_option_builder):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platform,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for dist in resolveds:
log(' %s' % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def assert_access_zipped_assets(distribution_helper_import):
test_executable = dedent("""
import os
{distribution_helper_import}
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
""".format(distribution_helper_import=distribution_helper_import))
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pb.add_source(None, 'my_package/__init__.py')
pb.add_source(None, 'my_package/submodule/__init__.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
process = PEX(pex,
interpreter=pb.interpreter).run(blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert process.returncode == 0
assert b'accessed\n' == stdout
return stderr
def build_pex(reqs, options, cache=None):
interpreters = None # Default to the current interpreter.
pex_python_path = options.python_path # If None, this will result in using $PATH.
# TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag.
if pex_python_path is None and (options.rc_file
or not ENV.PEX_IGNORE_RCFILES):
rc_variables = Variables(rc=options.rc_file)
pex_python_path = rc_variables.PEX_PYTHON_PATH
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed("Resolving interpreters", V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" %
full_path_or_basename)
return interp
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed("Resolving interpreters", V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
try:
interpreters = list(
iter_compatible_interpreters(
path=pex_python_path,
interpreter_constraints=constraints))
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Could not find a compatible interpreter."),
CANNOT_SETUP_INTERPRETER,
)
platforms = OrderedSet(options.platforms)
interpreters = interpreters or []
if options.platforms and options.resolve_local_platforms:
with TRACER.timed(
"Searching for local interpreters matching {}".format(
", ".join(map(str, platforms)))):
candidate_interpreters = OrderedSet(
iter_compatible_interpreters(path=pex_python_path))
candidate_interpreters.add(PythonInterpreter.get())
for candidate_interpreter in candidate_interpreters:
resolved_platforms = candidate_interpreter.supported_platforms.intersection(
platforms)
if resolved_platforms:
for resolved_platform in resolved_platforms:
TRACER.log("Resolved {} for platform {}".format(
candidate_interpreter, resolved_platform))
platforms.remove(resolved_platform)
interpreters.append(candidate_interpreter)
if platforms:
TRACER.log(
"Could not resolve a local interpreter for {}, will resolve only binary distributions "
"for {}.".format(
", ".join(map(str, platforms)),
"this platform"
if len(platforms) == 1 else "these platforms",
))
interpreter = (PythonInterpreter.latest_release_of_min_compatible_version(
interpreters) if interpreters else None)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
include_tools=options.include_tools or options.venv,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead.")
for directory in OrderedSet(options.sources_directory +
options.resources_directory):
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.unzip = options.unzip
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv
pex_info.venv_copies = options.venv_copies
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = InheritPath.for_value(options.inherit_path)
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
indexes = compute_indexes(options)
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(reqs +
options.requirement_files)):
if options.cache_ttl:
pex_warnings.warn(
"The --cache-ttl option is deprecated and no longer has any effect."
)
if options.headers:
pex_warnings.warn(
"The --header option is deprecated and no longer has any effect."
)
network_configuration = NetworkConfiguration(
retries=options.retries,
timeout=options.timeout,
proxy=options.proxy,
cert=options.cert,
client_cert=options.client_cert,
)
try:
if options.pex_repository:
with TRACER.timed("Resolving requirements from PEX {}.".format(
options.pex_repository)):
resolveds = resolve_from_pex(
pex=options.pex_repository,
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
network_configuration=network_configuration,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
manylinux=options.manylinux,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
indexes=indexes,
find_links=options.find_links,
resolver_version=ResolverVersion.for_value(
options.resolver_version),
network_configuration=network_configuration,
cache=cache,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors,
)
for resolved_dist in resolveds:
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(
resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(str(e))
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def build_pex(args, options, resolver_option_builder, interpreter=None):
if interpreter is None:
with TRACER.timed('Resolving interpreter', V=2):
interpreter = interpreter_from_options(options)
if interpreter is None:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
resolvables = [
Resolvable.get(arg, resolver_option_builder) for arg in args
]
for requirements_txt in options.requirement_files:
resolvables.extend(
requirements_from_file(requirements_txt, resolver_option_builder))
resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
if options.cache_dir:
resolver = CachingResolver(options.cache_dir, options.cache_ttl,
**resolver_kwargs)
else:
resolver = Resolver(**resolver_kwargs)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolver.resolve(resolvables)
except Unsatisfiable as e:
die(e)
for dist in resolveds:
log(' %s' % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def _build_pex(self, interpreter, path, vts):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
for vt in vts:
self._dump_sources(builder, vt.target)
builder.freeze()
def _build_requirements_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
dump_requirements(builder, interpreter, req_libs, self.context.log)
builder.freeze()
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].items():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def builder(shebang): pb = PEXBuilder() pb.set_shebang(shebang) return pb
def create_pex(self, pex_info=None):
"""Returns a wrapped pex that "merges" the other pexes via PEX_PATH."""
relevant_targets = self.context.targets(lambda tgt: isinstance(
tgt, (PythonRequirementLibrary, PythonTarget, Files)))
with self.invalidated(relevant_targets) as invalidation_check:
# If there are no relevant targets, we still go through the motions of resolving
# an empty set of requirements, to prevent downstream tasks from having to check
# for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter = self.context.products.get_data(PythonInterpreter)
path = os.path.join(self.workdir, str(interpreter.identity),
target_set_id)
extra_pex_paths = None
# Note that we check for the existence of the directory, instead of for invalid_vts,
# to cover the empty case.
if not os.path.isdir(path):
pexes = [
self.context.products.get_data(
ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(
GatherSources.PYTHON_SOURCES)
]
if self.extra_requirements():
extra_reqs = [
PythonRequirement(req_str)
for req_str in self.extra_requirements()
]
addr = Address.parse('{}_extra_reqs'.format(
self.__class__.__name__))
self.context.build_graph.inject_synthetic_target(
addr,
PythonRequirementLibrary,
requirements=extra_reqs)
# Add the extra requirements first, so they take precedence over any colliding version
# in the target set's dependency closure.
pexes = [
self.resolve_requirements(
[self.context.build_graph.get_target(addr)])
] + pexes
extra_pex_paths = [pex.path() for pex in pexes if pex]
if extra_pex_paths:
pex_info.merge_pex_path(':'.join(extra_pex_paths))
with safe_concurrent_creation(path) as safe_path:
builder = PEXBuilder(safe_path,
interpreter,
pex_info=pex_info)
# Add target interpreter compatibilities to pex info.
for rt in relevant_targets:
if has_python_sources(rt):
for constraint in rt.compatibility:
builder.add_interpreter_constraint(constraint)
builder.freeze()
return WrappedPEX(PEX(os.path.realpath(path), interpreter),
interpreter)
def build_pex(pex_filename: str, local_only: bool) -> str:
pex_builder = PEXBuilder(include_tools=True)
pex_builder.info.inherit_path = InheritPath.FALLBACK
pex_builder.set_entry_point('daml_dit_if.main:main')
pex_builder.set_shebang('/usr/bin/env python3')
platforms = [parsed_platform('current')]
if local_only:
LOG.warn('Local-only build. THIS DIT WILL NOT RUN IN DAML HUB.')
else:
platforms = [
*platforms,
parsed_platform('manylinux2014_x86_64-cp-38-cp38')
]
daml_dit_if_bundled = False
try:
if os.path.isfile(PYTHON_REQUIREMENT_FILE):
LOG.info(
f'Bundling dependencies from {PYTHON_REQUIREMENT_FILE}...')
requirement_files = [PYTHON_REQUIREMENT_FILE]
else:
LOG.info(
f'No dependency file found ({PYTHON_REQUIREMENT_FILE}), no dependencies will be bundled.'
)
requirement_files = []
resolveds = resolve_multi(requirements=[],
requirement_files=requirement_files,
platforms=platforms)
for resolved_dist in resolveds:
if resolved_dist.distribution.project_name == IF_PROJECT_NAME \
and not daml_dit_if_bundled:
LOG.warn(
f'Bundling {IF_PROJECT_NAME} in output DIT file. This will'
f' override the version provided by Daml Hub, potentially'
f' compromising compatibility of this integration with'
f' future updates to Daml Hub. Use this option with caution.'
)
daml_dit_if_bundled = True
LOG.debug("req: %s", resolved_dist.distribution)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
LOG.info("direct_req: %s", resolved_dist.direct_requirement)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_requirement(resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(f'Unsatifiable dependency error: {e}')
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
LOG.debug("Adding source file: %r, %r", src_file_path,
dst_path)
fn(src_file_path, dst_path)
walk_and_do(pex_builder.add_source, 'src/')
pex_builder.freeze(bytecode_compile=True)
# Entry point verification is disabled because ddit does not
# formally depend on the integration framework, and it is not
# necessarily available at integration build time. Because entry
# point verification happens in ddit's environment and the
# entrypoint is in the framework, this causes entry point
# verification to fail unless some other agent has installed
# daml-dit-if into ddit's environment.
#
# Virtual environments provide a way to work around this (and are
# used in 'ddit run') but the PEX API does not allow a virtual
# environment to be specified at build time. If this ever changes,
# the build subcommand should be modified to prepare a virtual
# enviroment for the build that contains the appropriate version
# of daml-dit-if and entrypoint verification should be re-enabled.
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=False)
LOG.info('Building intermediate PEX file...')
LOG.debug('PEX info: %r', pex_builder.info)
pex_builder.build(pex_filename,
bytecode_compile=True,
deterministic_timestamp=True)
if daml_dit_if_bundled:
return 'python-direct'
else:
return 'python-direct-hub-if'
def test_prex_builder_script_from_pex_path(tmpdir):
# type: (Any) -> None
pex_with_script = os.path.join(str(tmpdir), "script.pex")
with built_wheel(
name="my_project",
entry_points={
"console_scripts":
["my_app = my_project.my_module:do_something"]
},
) as my_whl:
pb = PEXBuilder()
pb.add_dist_location(my_whl)
pb.build(pex_with_script)
pex_file = os.path.join(str(tmpdir), "app.pex")
pb = PEXBuilder()
pb.info.pex_path = pex_with_script
pb.set_script("my_app")
pb.build(pex_file)
assert "hello world!\n" == subprocess.check_output(
args=[pex_file]).decode("utf-8")
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir,
CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' %
(sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(
sdist,
os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def _build_pex(self, interpreter, path, targets):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
for target in targets:
dump_sources(builder, target, self.context.log)
builder.freeze()
def yield_pex_builder(zip_safe=True, interpreter=None):
# type: (bool, Optional[PythonInterpreter]) -> Iterator[PEXBuilder]
with temporary_dir() as td, make_bdist("p1", zip_safe=zip_safe, interpreter=interpreter) as p1:
pb = PEXBuilder(path=td, interpreter=interpreter)
pb.add_dist_location(p1.location)
yield pb
def build_pex(reqs, options):
interpreters = None # Default to the current interpreter.
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(
full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' %
full_path_or_basename)
return interpreter
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed('Resolving interpreters', V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
if options.rc_file or not ENV.PEX_IGNORE_RCFILES:
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', None)
else:
pex_python_path = None
interpreters = list(
iter_compatible_interpreters(pex_python_path, constraints))
if not interpreters:
die('Could not find compatible interpreter',
CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters) if interpreters else None
pex_builder = PEXBuilder(path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
# NB: `None` means use the default (pypi) index, `[]` means use no indexes.
indexes = None
if options.indexes != [_PYPI] and options.indexes is not None:
indexes = [str(index) for index in options.indexes]
with TRACER.timed(
'Resolving distributions ({})'.format(reqs +
options.requirement_files)):
try:
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=options.platforms,
indexes=indexes,
find_links=options.find_links,
cache=options.cache_dir,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors)
for resolved_dist in resolveds:
log(' %s -> %s' %
(resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def builder(shebang):
# type: (str) -> PEXBuilder
pb = PEXBuilder()
pb.set_shebang(shebang)
return pb
def assert_force_local_implicit_ns_packages_issues_598(
interpreter=None, requirements=(), create_ns_packages=True
):
def create_foo_bar_setup(name, **extra_args):
# type: (str, **Any) -> str
setup_args = dict(name=name, version="0.0.1", packages=["foo", "foo.bar"])
if create_ns_packages:
setup_args.update(namespace_packages=["foo", "foo.bar"])
if requirements:
setup_args.update(install_requires=list(requirements))
setup_args.update(extra_args)
return dedent(
"""
from setuptools import setup
setup(**{setup_args!r})
""".format(
setup_args=setup_args
)
)
def with_foo_bar_ns_packages(content):
# type: (Dict[str, str]) -> Dict[str, str]
ns_packages = (
{
os.path.join(
pkg, "__init__.py"
): '__import__("pkg_resources").declare_namespace(__name__)'
for pkg in ("foo", "foo/bar")
}
if create_ns_packages
else {}
)
ns_packages.update(content)
return ns_packages
content1 = with_foo_bar_ns_packages(
{
"foo/bar/spam.py": "identify = lambda: 42",
"setup.py": create_foo_bar_setup("foo-bar-spam"),
}
)
content2 = with_foo_bar_ns_packages(
{
"foo/bar/eggs.py": dedent(
"""
# NB: This only works when this content is unpacked loose on the filesystem!
def read_self():
with open(__file__) as fp:
return fp.read()
"""
)
}
)
content3 = with_foo_bar_ns_packages(
{
"foobaz": dedent(
"""\
#!python
import sys
from foo.bar import baz
sys.exit(baz.main())
"""
),
"foo/bar/baz.py": dedent(
"""
import sys
from foo.bar import eggs, spam
def main():
assert len(eggs.read_self()) > 0
return spam.identify()
"""
),
"setup.py": create_foo_bar_setup("foo-bar-baz", scripts=["foobaz"]),
}
)
def add_requirements(builder, cache):
# type: (PEXBuilder, str) -> None
for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter):
builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
builder.add_requirement(resolved_dist.direct_requirement)
def add_wheel(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
dist = WheelBuilder(project, interpreter=builder.interpreter).bdist()
builder.add_dist_location(dist)
def add_sources(builder, content):
# type: (PEXBuilder, Dict[str, str]) -> None
with temporary_content(content) as project:
for path in content.keys():
builder.add_source(os.path.join(project, path), path)
with temporary_dir() as root, temporary_dir() as cache:
pex_info1 = PexInfo.default()
pex_info1.zip_safe = False
pex1 = os.path.join(root, "pex1.pex")
builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1)
add_requirements(builder1, cache)
add_wheel(builder1, content1)
add_sources(builder1, content2)
builder1.build(pex1)
pex_info2 = PexInfo.default()
pex_info2.pex_path = pex1
pex2 = os.path.join(root, "pex2")
builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2)
add_requirements(builder2, cache)
add_wheel(builder2, content3)
builder2.set_script("foobaz")
builder2.freeze()
assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE="9"))
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]
if options.interpreter_constraint:
# NB: options.python and interpreter constraints cannot be used together, so this will not
# affect usages of the interpreter(s) specified by the "--python" command line flag.
constraints = options.interpreter_constraint
validate_constraints(constraints)
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
interpreters = find_compatible_interpreters(pex_python_path, constraints)
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt,
builder=resolver_option_builder,
interpreter=interpreter))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
builder=resolver_option_builder,
interpreter=interpreter):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platforms,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
build_properties = PexInfo.make_build_properties()
if self.get_options().include_run_information:
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=tmpdir,
interpreter=interpreter,
pex_info=pex_info,
copy=True),
log=self.context.log)
if binary_tgt.shebang:
self.context.log.info(
'Found Python binary target {} with customized shebang, using it: {}'
.format(binary_tgt.name, binary_tgt.shebang))
pex_builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug(
f'No customized shebang found for {binary_tgt.name}')
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
constraint_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
if is_python_target(tgt):
constraint_tgts.append(tgt)
# Add interpreter compatibility constraints to pex info. Note that we only add the constraints for the final
# binary target itself, not its dependencies. The upstream interpreter selection tasks will already validate that
# there are no compatibility conflicts among the dependencies and target. If the binary target does not have
# `compatibility` in its BUILD entry, the global --python-setup-interpreter-constraints will be used.
pex_builder.add_interpreter_constraints_from([binary_tgt])
# Dump everything into the builder's chroot.
for tgt in source_tgts:
pex_builder.add_sources_from(tgt)
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
self._python_native_code_settings.check_build_for_current_platform_only(
self.context.targets())
pex_builder.add_requirement_libs_from(
req_tgts, platforms=binary_tgt.platforms)
# Build the .pex file.
pex_path = os.path.join(results_dir, f'{binary_tgt.name}.pex')
pex_builder.build(pex_path)
return pex_path
