def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider):
# Short-circuit if there is a local copy
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
context = Context.get()
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]
with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2):
for link in links:
with TRACER.timed('Fetching %s' % link, V=3):
sdist = context.fetch(link)
with TRACER.timed('Installing %s' % link, V=3):
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(
os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
return EggPackage(target_location)
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreter', V=2):
interpreter = interpreter_from_options(options)
if interpreter is None:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt, resolver_option_builder):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
if options.cache_dir:
resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
else:
resolver = Resolver(**resolver_kwargs)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolver.resolve(resolvables)
except Unsatisfiable as e:
die(e)
for dist in resolveds:
log(' %s' % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def build_pex(args, options, resolver_option_builder, interpreter=None):
if interpreter is None:
with TRACER.timed("Resolving interpreter", V=2):
interpreter = interpreter_from_options(options)
if interpreter is None:
die("Could not find compatible interpreter", CANNOT_SETUP_INTERPRETER)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
if options.cache_dir:
resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
else:
resolver = Resolver(**resolver_kwargs)
with TRACER.timed("Resolving distributions"):
try:
resolveds = resolver.resolve(resolvables)
except Unsatisfiable as e:
die(e)
for dist in resolveds:
log(" %s" % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def build(self, package, options):
context = options.get_context()
translator = options.get_translator(self._interpreter, self._supported_tags)
with TRACER.timed('Fetching %s' % package.url, V=2):
local_package = Package.from_href(context.fetch(package))
if local_package is None:
raise Untranslateable('Could not fetch package %s' % package)
with TRACER.timed('Translating %s into distribution' % local_package.local_path, V=2):
dist = translator.translate(local_package)
if dist is None:
raise Untranslateable('Package %s is not translateable by %s' % (package, translator))
if not distribution_compatible(dist, self._supported_tags):
raise Untranslateable(
'Could not get distribution for %s on platform %s.' % (package, self._platform))
return dist
def build_pex(args):
with TRACER.timed('Resolving interpreter', V=2):
interpreter = _establish_interpreter(args)
if interpreter is None:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE)
pex_info = pex_builder.info
pex_info.zip_safe = False
pex_info.always_write_cache = True
pex_info.inherit_path = False
resolver_option_builder = _establish_resolver_options(args)
reqs = args.reqs
resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs]
for requirements_txt in args.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
resolver_kwargs = dict(interpreter=interpreter, platform=args.platform)
_add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder)
if not args.disable_cache:
resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs)
else:
resolver = Resolver(**resolver_kwargs)
resolveds = []
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolver.resolve(resolvables)
except Unsatisfiable as exception:
die(exception)
for dist in resolveds:
log(' %s' % dist, verbose=args.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.set_entry_point('spex:spex')
if args.python_shebang:
pex_builder.set_shebang(args.python_shebang)
return pex_builder
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def activate(self):
if not self._activated:
with TRACER.timed('Activating PEX virtual environment from %s' % self._pex):
self._working_set = self._activate()
self._activated = True
return self._working_set
def run(self):
if self._installed is not None:
return self._installed
with TRACER.timed('Installing %s' % self._install_tmp, V=2):
env = self._interpreter.sanitized_environment()
mixins = OrderedSet(['setuptools'] + self.mixins)
env['PYTHONPATH'] = os.pathsep.join(third_party.expose(mixins))
env['__PEX_UNVENDORED__'] = '1'
command = [self._interpreter.binary, '-s', '-'] + self.setup_command()
try:
Executor.execute(command,
env=env,
cwd=self._source_dir,
stdin_payload=self.setup_py_wrapper.encode('ascii'))
self._installed = True
except Executor.NonZeroExit as e:
self._installed = False
name = os.path.basename(self._source_dir)
print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr)
print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr)
return self._installed
return self._installed
def main(args=None):
parser = create_parser()
args = parser.parse_args(args)
if args.build_distro:
if not args.spark_home:
die("No spark home given but building a distribution")
spark_home = os.path.realpath(os.path.abspath(args.spark_home))
if not os.path.exists(spark_home):
die("No spark home given but building a distribution")
spark_name = os.path.basename(spark_home)
args.spark_home = spark_home
args.spark_name = spark_name
else:
spark_home = None
spark_name = None
spex_name = args.spex_name
spex_file = spex_name + '.spex'
with ENV.patch(PEX_VERBOSE=str(args.verbosity)):
with TRACER.timed('Building spex'):
with TRACER.timed('Building pex'):
pex_builder = build_pex(args)
with dump_args_as_config(args) as cfg:
pex_builder.add_resource(cfg, 'SPEX-INFO')
log('Saving PEX file to %s' % spex_file, verbose=args.verbosity)
tmp_name = args.spex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, spex_file)
if args.build_distro:
with TRACER.timed('Building spark package'):
spark_distro = uber_distro_location(spark_name)
establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name)
log('Spark package built')
with TRACER.timed('Building full distribution'):
create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args)
log('Saved full distribution to %s' % spark_distro)
return 0
def load_internal_cache(cls, pex, pex_info):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(pex, pex_info.internal_cache)
with TRACER.timed('Searching dependency cache: %s' % internal_cache, V=2):
if os.path.isdir(pex):
for dist in find_distributions(internal_cache):
yield dist
else:
for dist in itertools.chain(*cls.write_zipped_internal_cache(pex, pex_info)):
yield dist
def _activate(self):
self.update_candidate_distributions(self.load_internal_cache(self._pex, self._pex_info))
if not self._pex_info.zip_safe and os.path.isfile(self._pex):
explode_dir = self.force_local(pex_file=self._pex, pex_info=self._pex_info)
self.update_module_paths(pex_file=self._pex, explode_dir=explode_dir)
all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements]
working_set = WorkingSet([])
resolved = self._resolve(working_set, all_reqs)
for dist in resolved:
with TRACER.timed('Activating %s' % dist, V=2):
working_set.add(dist)
if self._inherit_path == "fallback":
# Prepend location to sys.path.
#
# This ensures that bundled versions of libraries will be used before system-installed
# versions, in case something is installed in both, helping to favor hermeticity in
# the case of non-hermetic PEX files (i.e. those with inherit_path=True).
#
# If the path is not already in sys.path, site.addsitedir will append (not prepend)
# the path to sys.path. But if the path is already in sys.path, site.addsitedir will
# leave sys.path unmodified, but will do everything else it would do. This is not part
# of its advertised contract (which is very vague), but has been verified to be the
# case by inspecting its source for both cpython 2.7 and cpython 3.7.
sys.path.insert(0, dist.location)
else:
sys.path.append(dist.location)
with TRACER.timed('Adding sitedir', V=2):
site.addsitedir(dist.location)
self._declare_namespace_packages(resolved)
return working_set
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!')
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def maybe_reexec_pex(compatibility_constraints):
"""
Handle environment overrides for the Python interpreter to use when executing this pex.
This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO
metadata. If PEX_PYTHON is set in a pexrc, it attempts to obtain the binary location of the
interpreter specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for
a matching interpreter in accordance with the interpreter constraints. If both variables are
present in a pexrc, this function gives precedence to PEX_PYTHON_PATH and errors out if no
compatible interpreters can be found on said path.
If neither variable is set, we fall back to plain PEX execution using PATH searching or the
currently executing interpreter. If compatibility constraints are used, we match those constraints
against these interpreters.
:param compatibility_constraints: list of requirements-style strings that constrain the
Python interpreter to re-exec this pex with.
"""
if os.environ.pop('SHOULD_EXIT_BOOTSTRAP_REEXEC', None):
# We've already been here and selected an interpreter. Continue to execution.
return
target = None
with TRACER.timed('Selecting runtime interpreter based on pexrc', V=3):
if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH:
# preserve PEX_PYTHON re-exec for backwards compatibility
# TODO: Kill this off completely in favor of PEX_PYTHON_PATH
# https://github.com/pantsbuild/pex/issues/431
target = _select_pex_python_interpreter(ENV.PEX_PYTHON,
compatibility_constraints=compatibility_constraints)
elif ENV.PEX_PYTHON_PATH:
target = _select_interpreter(pex_python_path=ENV.PEX_PYTHON_PATH,
compatibility_constraints=compatibility_constraints)
elif compatibility_constraints:
# Apply constraints to target using regular PATH
target = _select_interpreter(compatibility_constraints=compatibility_constraints)
if target and os.path.realpath(target) != os.path.realpath(sys.executable):
cmdline = [target] + sys.argv
TRACER.log('Re-executing: cmdline="%s", sys.executable="%s", PEX_PYTHON="%s", '
'PEX_PYTHON_PATH="%s", COMPATIBILITY_CONSTRAINTS="%s"'
% (cmdline, sys.executable, ENV.PEX_PYTHON, ENV.PEX_PYTHON_PATH,
compatibility_constraints))
ENV.delete('PEX_PYTHON')
ENV.delete('PEX_PYTHON_PATH')
os.environ['SHOULD_EXIT_BOOTSTRAP_REEXEC'] = '1'
os.execve(target, cmdline, ENV.copy())
def download_distributions(self, dest=None, max_parallel_jobs=None):
if not self.requirements and not self.requirement_files:
# Nothing to resolve.
return None
dest = dest or safe_mkdtemp()
spawn_download = functools.partial(self._spawn_download, dest)
with TRACER.timed('Resolving for:\n {}'.format('\n '.join(map(str, self.targets)))):
return list(
execute_parallel(
inputs=self.targets,
spawn_func=spawn_download,
raise_type=Unsatisfiable,
max_jobs=max_parallel_jobs
)
)
def _force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache,
pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
with atomic_directory(explode_dir) as explode_tmp:
if explode_tmp:
with TRACER.timed('Unzipping %s' % pex_file):
with open_zip(pex_file) as pex_zip:
pex_files = (
x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR)
and not x.startswith(pex_info.internal_cache))
pex_zip.extractall(explode_tmp, pex_files)
return explode_dir
def _load_internal_cache(self):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(self._pex, self._pex_info.internal_cache)
with TRACER.timed("Searching dependency cache: %s" % internal_cache,
V=2):
if len(self._pex_info.distributions) == 0:
# We have no .deps to load.
return
if os.path.isdir(self._pex):
for distribution_name in self._pex_info.distributions:
yield DistributionHelper.distribution_from_path(
os.path.join(internal_cache, distribution_name))
else:
with open_zip(self._pex) as zf:
for dist in self._write_zipped_internal_cache(zf):
yield dist
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def build(self,
filename,
bytecode_compile=True,
deterministic_timestamp=False):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
:param bytecode_compile: If True, precompile .py files into .pyc files.
:param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
self.freeze(bytecode_compile=bytecode_compile)
tmp_zip = filename + "~"
try:
os.unlink(tmp_zip)
self._logger.warning(
"Previous binary unexpectedly exists, cleaning: {}".format(
tmp_zip))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with safe_open(tmp_zip, "ab") as pexfile:
assert os.path.getsize(pexfile.name) == 0
pexfile.write(to_bytes("{}\n".format(self._shebang)))
with TRACER.timed("Zipping PEX file."):
self._chroot.zip(
tmp_zip,
mode="a",
deterministic_timestamp=deterministic_timestamp,
# When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions
# as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since
# those installed wheels reside in a shared cache, they can be in-use by other
# processes and so their code may be in the process of being bytecode compiled as we
# attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary
# pyc files that we should avoid copying since they are unuseful and inherently
# racy.
exclude_file=is_pyc_temporary_file,
)
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmp_zip, filename)
chmod_plus_x(filename)
def _resolve(self, working_set, reqs):
reqs = reqs[:]
unresolved_reqs = set()
resolveds = set()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(
set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
'Skipping activation of `%s` due to environment marker de-selection'
% req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
unresolved_reqs.add(e.req.project_name)
if e.requirers:
unresolved_reqs.update(e.requirers)
unresolved_reqs = set([req.lower() for req in unresolved_reqs])
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist in self._pex_info.distributions:
TRACER.log(' - %s' % dist)
if not self._pex_info.ignore_errors:
die('Failed to execute PEX file, missing %s compatible dependencies for:\n%s'
% (Platform.current(), '\n'.join(
str(r) for r in unresolved_reqs)))
return resolveds
def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(
filter(None,
(filename[prefix_length:].split('/')[0]
for filename in zf.namelist()
if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join(
[pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(
distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(
pex_info.install_cache,
'%s.%s' % (distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(
cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(
os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(
dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path,
cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def _resolve(self, working_set, reqs):
reqs = reqs[:]
unresolved_reqs = set()
resolveds = set()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
unresolved_reqs.add(e.req.project_name)
if e.requirers:
unresolved_reqs.update(e.requirers)
unresolved_reqs = set([req.lower() for req in unresolved_reqs])
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist in self._pex_info.distributions:
TRACER.log(' - %s' % dist)
if not self._pex_info.ignore_errors:
die(
'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % (
Platform.current(),
'\n'.join(str(r) for r in unresolved_reqs)
)
)
return resolveds
def explode_code(
self,
dest_dir, # type: str
exclude=(), # type: Container[str]
):
# type: (...) -> Iterable[Tuple[str, str]]
with TRACER.timed("Unzipping {}".format(self._pex)):
with open_zip(self._pex) as pex_zip:
pex_files = (
name for name in pex_zip.namelist()
if not name.startswith(pex_builder.BOOTSTRAP_DIR)
and not name.startswith(self._pex_info.internal_cache)
and name not in exclude)
pex_zip.extractall(dest_dir, pex_files)
return [(
"{pex_file}:{zip_path}".format(pex_file=self._pex,
zip_path=f),
os.path.join(dest_dir, f),
) for f in pex_files]
def run(self):
if self._installed is not None:
return self._installed
with TRACER.timed('Installing %s' % self._install_tmp, V=2):
command = [self._interpreter.binary, '-sE', '-'] + self._setup_command()
try:
Executor.execute(command,
env=self._interpreter.sanitized_environment(),
cwd=self._source_dir,
stdin_payload=self.bootstrap_script.encode('ascii'))
self._installed = True
except Executor.NonZeroExit as e:
self._installed = False
name = os.path.basename(self._source_dir)
print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr)
print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr)
return self._installed
return self._installed
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def fetch(self, link, into=None):
"""Fetch the binary content associated with the link and write to a file.
:param link: The :class:`Link` to fetch.
:keyword into: If specified, write into the directory ``into``. If ``None``, creates a new
temporary directory that persists for the duration of the interpreter.
"""
target = os.path.join(into or safe_mkdtemp(), link.filename)
if os.path.exists(target):
# Assume that if the local file already exists, it is safe to use.
return target
with TRACER.timed('Fetching %s' % link.url, V=2):
target_tmp = '%s.%s' % (target, uuid.uuid4())
with contextlib.closing(self.open(link)) as in_fp:
with safe_open(target_tmp, 'wb') as out_fp:
shutil.copyfileobj(in_fp, out_fp)
os.rename(target_tmp, target)
return target
def _load_internal_cache(cls, pex, pex_info):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(pex, pex_info.internal_cache)
with TRACER.timed("Searching dependency cache: %s" % internal_cache, V=2):
if len(pex_info.distributions) == 0:
# We have no .deps to load.
return
if os.path.isdir(pex):
search_path = [
os.path.join(internal_cache, dist_chroot)
for dist_chroot in os.listdir(internal_cache)
]
internal_env = Environment(search_path=search_path)
for dist_name in internal_env:
for dist in internal_env[dist_name]:
yield dist
else:
with open_zip(pex) as zf:
for dist in cls._write_zipped_internal_cache(zf, pex_info):
yield dist
def fetch(self, link, into=None):
"""Fetch the binary content associated with the link and write to a file.
:param link: The :class:`Link` to fetch.
:keyword into: If specified, write into the directory ``into``. If ``None``, creates a new
temporary directory that persists for the duration of the interpreter.
"""
target = os.path.join(into or safe_mkdtemp(), link.filename)
if os.path.exists(target):
# Assume that if the local file already exists, it is safe to use.
return target
with TRACER.timed('Fetching %s' % link.url, V=2):
target_tmp = '%s.%s' % (target, uuid.uuid4())
with contextlib.closing(self.open(link)) as in_fp:
with safe_open(target_tmp, 'wb') as out_fp:
shutil.copyfileobj(in_fp, out_fp)
os.rename(target_tmp, target)
return target
def find(cls):
# type: () -> Optional[Pyenv]
"""Finds the active pyenv installation if any."""
with TRACER.timed("Searching for pyenv root...", V=3):
pyenv_root = os.environ.get("PYENV_ROOT", "")
if not pyenv_root:
for path_entry in os.environ.get("PATH", "").split(os.pathsep):
pyenv_exe = os.path.join(path_entry, "pyenv")
if is_exe(pyenv_exe):
process = subprocess.Popen(args=[pyenv_exe, "root"], stdout=subprocess.PIPE)
stdout, _ = process.communicate()
if process.returncode == 0:
pyenv_root = str(stdout).strip()
break
if pyenv_root:
pyenv = cls(pyenv_root)
TRACER.log("A pyenv installation was found: {}".format(pyenv), V=6)
return pyenv
TRACER.log("No pyenv installation was found.", V=6)
return None
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def _establish_interpreter(args):
if args.python:
if os.path.exists(args.python):
interpreter = PythonInterpreter.from_binary(args.python)
else:
interpreter = PythonInterpreter.from_env(args.python)
if interpreter is None:
die('Failed to find interpreter: %s' % args.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, args.interpreter_cache_dir, args.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and args.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def active_versions(self, search_dir=None):
# type: (Optional[str]) -> Tuple[str, ...]
"""Reports the active pyenv versions for the given starting search directory or $PWD."""
source_and_versions = None # type: Optional[Tuple[str, Iterable[str]]]
# See: https://github.com/pyenv/pyenv#choosing-the-python-version
with TRACER.timed("Finding {} active versions...".format(self), V=6):
shell_version = os.environ.get("PYENV_VERSION")
if shell_version:
source_and_versions = (
"PYENV_VERSION={}".format(shell_version),
shell_version.split(":"),
)
else:
cwd = search_dir if search_dir is not None else os.getcwd()
TRACER.log("Looking for pyenv version files starting from {}.".format(cwd), V=6)
local_version = self._find_local_version_file(search_dir=cwd)
if local_version:
source_and_versions = (local_version, self._read_pyenv_versions(local_version))
else:
global_version = os.path.join(self.root, "version")
if os.path.exists(global_version):
source_and_versions = (
global_version,
self._read_pyenv_versions(global_version),
)
if source_and_versions:
source, versions = source_and_versions
TRACER.log("Found active versions in {}: {}".format(source, versions), V=6)
return tuple(versions)
TRACER.log("Found no active pyenv versions.", V=6)
return ()
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die("Failed to find interpreter: %s" % options.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed("Setting up interpreter %s" % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and options.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
interpreter = None
if python_interpreter:
if os.path.exists(python_interpreter):
interpreter = PythonInterpreter.from_binary(python_interpreter)
else:
interpreter = PythonInterpreter.from_env(python_interpreter)
if interpreter is None:
die('Failed to find interpreter: %s' % python_interpreter)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
interpreter = None
if python_interpreter:
if os.path.exists(python_interpreter):
interpreter = PythonInterpreter.from_binary(python_interpreter)
else:
interpreter = PythonInterpreter.from_env(python_interpreter)
if interpreter is None:
die('Failed to find interpreter: %s' % python_interpreter)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def build_pex(reqs, options, cache=None):
interpreters = None # Default to the current interpreter.
pex_python_path = options.python_path # If None, this will result in using $PATH.
# TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag.
if pex_python_path is None and (options.rc_file
or not ENV.PEX_IGNORE_RCFILES):
rc_variables = Variables(rc=options.rc_file)
pex_python_path = rc_variables.PEX_PYTHON_PATH
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed("Resolving interpreters", V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" %
full_path_or_basename)
return interp
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed("Resolving interpreters", V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
try:
interpreters = list(
iter_compatible_interpreters(
path=pex_python_path,
interpreter_constraints=constraints))
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Could not find a compatible interpreter."),
CANNOT_SETUP_INTERPRETER,
)
platforms = OrderedSet(options.platforms)
interpreters = interpreters or []
if options.platforms and options.resolve_local_platforms:
with TRACER.timed(
"Searching for local interpreters matching {}".format(
", ".join(map(str, platforms)))):
candidate_interpreters = OrderedSet(
iter_compatible_interpreters(path=pex_python_path))
candidate_interpreters.add(PythonInterpreter.get())
for candidate_interpreter in candidate_interpreters:
resolved_platforms = candidate_interpreter.supported_platforms.intersection(
platforms)
if resolved_platforms:
for resolved_platform in resolved_platforms:
TRACER.log("Resolved {} for platform {}".format(
candidate_interpreter, resolved_platform))
platforms.remove(resolved_platform)
interpreters.append(candidate_interpreter)
if platforms:
TRACER.log(
"Could not resolve a local interpreter for {}, will resolve only binary distributions "
"for {}.".format(
", ".join(map(str, platforms)),
"this platform"
if len(platforms) == 1 else "these platforms",
))
interpreter = (PythonInterpreter.latest_release_of_min_compatible_version(
interpreters) if interpreters else None)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
include_tools=options.include_tools or options.venv,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead.")
for directory in OrderedSet(options.sources_directory +
options.resources_directory):
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.unzip = options.unzip
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv
pex_info.venv_copies = options.venv_copies
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = InheritPath.for_value(options.inherit_path)
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
indexes = compute_indexes(options)
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(reqs +
options.requirement_files)):
if options.cache_ttl:
pex_warnings.warn(
"The --cache-ttl option is deprecated and no longer has any effect."
)
if options.headers:
pex_warnings.warn(
"The --header option is deprecated and no longer has any effect."
)
network_configuration = NetworkConfiguration(
retries=options.retries,
timeout=options.timeout,
proxy=options.proxy,
cert=options.cert,
client_cert=options.client_cert,
)
try:
if options.pex_repository:
with TRACER.timed("Resolving requirements from PEX {}.".format(
options.pex_repository)):
resolveds = resolve_from_pex(
pex=options.pex_repository,
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
network_configuration=network_configuration,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
manylinux=options.manylinux,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
indexes=indexes,
find_links=options.find_links,
resolver_version=ResolverVersion.for_value(
options.resolver_version),
network_configuration=network_configuration,
cache=cache,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors,
)
for resolved_dist in resolveds:
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(
resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(str(e))
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options = parser.parse_args(args=args)
# Ensure the TMPDIR is an absolute path (So subprocesses that change CWD can find it) and
# that it exists.
tmpdir = os.path.realpath(options.tmpdir)
if not os.path.exists(tmpdir):
die("The specified --tmpdir does not exist: {}".format(tmpdir))
if not os.path.isdir(tmpdir):
die("The specified --tmpdir is not a directory: {}".format(tmpdir))
tempfile.tempdir = os.environ["TMPDIR"] = tmpdir
if options.cache_dir:
pex_warnings.warn(
"The --cache-dir option is deprecated, use --pex-root instead.")
if options.pex_root and options.cache_dir != options.pex_root:
die("Both --cache-dir and --pex-root were passed with conflicting values. "
"Just set --pex-root.")
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn(
"The pex root has been set via {via} but --disable-cache is also set. "
"Ignoring {via} and disabling caches.".format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root("--cache-dir")
elif options.pex_root:
warn_ignore_pex_root("--pex-root")
elif os.environ.get("PEX_ROOT"):
warn_ignore_pex_root("PEX_ROOT")
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_repository and (options.indexes or options.find_links):
die('The "--pex-repository" option cannot be used together with the "--index" or '
'"--find-links" options.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=pex_root,
TMPDIR=tmpdir) as patched_env:
with TRACER.timed("Building pex"):
pex_builder = build_pex(options.requirements,
options,
cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
interpreter = pex_builder.interpreter
pex = PEX(pex_builder.path(),
interpreter=interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log("Saving PEX file to %s" % options.pex_name,
V=options.verbosity)
pex_builder.build(
options.pex_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time,
)
if options.seed != Seed.NONE:
seed_info = seed_cache(options,
pex,
verbose=options.seed == Seed.VERBOSE)
print(seed_info)
else:
if not _compatible_with_current_platform(interpreter,
options.platforms):
log("WARNING: attempting to run PEX with incompatible platforms!",
V=1)
log(
"Running on platform {} but built for {}".format(
interpreter.platform,
", ".join(map(str, options.platforms))),
V=1,
)
log(
"Running PEX file at %s with args %s" %
(pex_builder.path(), cmdline),
V=options.verbosity,
)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def update_candidate_distributions(self, distribution_iter):
for dist in distribution_iter:
if self.can_add(dist):
with TRACER.timed('Adding %s' % dist, V=2):
self.add(dist)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(
options.requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
bdist = installer(source_dir).bdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_dist_location(bdist)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]
if options.interpreter_constraint:
# NB: options.python and interpreter constraints cannot be used together, so this will not
# affect usages of the interpreter(s) specified by the "--python" command line flag.
constraints = options.interpreter_constraint
validate_constraints(constraints)
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
interpreters = find_compatible_interpreters(pex_python_path, constraints)
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt,
builder=resolver_option_builder,
interpreter=interpreter))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
builder=resolver_option_builder,
interpreter=interpreter):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platforms,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def _resolve(self, working_set, reqs):
environment = self._target_interpreter_env.copy()
environment["extra"] = list(set(itertools.chain(*(req.extras for req in reqs))))
reqs_by_key = OrderedDict()
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
"Skipping activation of `%s` due to environment marker de-selection" % req
)
continue
reqs_by_key.setdefault(req.key, []).append(req)
unresolved_reqs = OrderedDict()
resolveds = OrderedSet()
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for key, reqs in reqs_by_key.items():
with TRACER.timed("Resolving {} from {}".format(key, reqs), V=2):
# N.B.: We resolve the bare requirement with no version specifiers since the resolve process
# used to build this pex already did so. There may be multiple distributions satisfying any
# particular key (e.g.: a Python 2 specific version and a Python 3 specific version for a
# multi-python PEX) and we want the working set to pick the most appropriate one.
req = Requirement.parse(key)
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log("Failed to resolve a requirement: %s" % e)
requirers = unresolved_reqs.setdefault(e.req, OrderedSet())
if e.requirers:
for requirer in e.requirers:
requirers.update(reqs_by_key[requirer])
if unresolved_reqs:
TRACER.log("Unresolved requirements:")
for req in unresolved_reqs:
TRACER.log(" - %s" % req)
TRACER.log("Distributions contained within this pex:")
distributions_by_key = defaultdict(list)
if not self._pex_info.distributions:
TRACER.log(" None")
else:
for dist_name, dist_digest in self._pex_info.distributions.items():
TRACER.log(" - %s" % dist_name)
distribution = DistributionHelper.distribution_from_path(
path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name)
)
distributions_by_key[distribution.as_requirement().key].append(distribution)
if not self._pex_info.ignore_errors:
items = []
for index, (requirement, requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ""
if requirers:
rendered_requirers = ("\n Required by:" "\n {requirers}").format(
requirers="\n ".join(map(str, requirers))
)
items.append(
"{index: 2d}: {requirement}"
"{rendered_requirers}"
"\n But this pex only contains:"
"\n {distributions}".format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
distributions="\n ".join(
os.path.basename(d.location)
for d in distributions_by_key[requirement.key]
),
)
)
die(
"Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}".format(
platform=self._interpreter.platform, items="\n".join(items)
)
)
return resolveds
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]
if options.interpreter_constraint:
# NB: options.python and interpreter constraints cannot be used together, so this will not
# affect usages of the interpreter(s) specified by the "--python" command line flag.
constraints = options.interpreter_constraint
validate_constraints(constraints)
if options.rc_file or not ENV.PEX_IGNORE_RCFILES:
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
else:
pex_python_path = ""
interpreters = find_compatible_interpreters(pex_python_path, constraints)
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt,
builder=resolver_option_builder,
interpreter=interpreter))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
builder=resolver_option_builder,
interpreter=interpreter):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platforms,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def main():
pparser, resolver_options_builder = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
if poptions.pex_root:
ENV.set('PEX_ROOT', poptions.pex_root)
else:
poptions.pex_root = ENV.PEX_ROOT
if poptions.cache_dir:
poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir)
poptions.interpreter_cache_dir = pexbin.make_relative_to_root(
poptions.interpreter_cache_dir)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions,
resolver_options_builder)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
v=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, poptions.pex_name)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' % (sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(sdist, os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(
requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.cache_dir:
pex_warnings.warn('The --cache-dir option is deprecated, use --pex-root instead.')
if options.pex_root and options.cache_dir != options.pex_root:
die('Both --cache-dir and --pex-root were passed with conflicting values. '
'Just set --pex-root.')
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn('The pex root has been set via {via} but --disable-cache is also set. '
'Ignoring {via} and disabling caches.'.format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root('--cache-dir')
elif options.pex_root:
warn_ignore_pex_root('--pex-root')
elif os.environ.get('PEX_ROOT'):
warn_ignore_pex_root('PEX_ROOT')
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root) as patched_env:
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!', V=1)
log('Running on platform {} but built for {}'
.format(Platform.current(), ', '.join(map(str, options.platforms))), V=1)
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def build_pex(reqs, options):
interpreters = None # Default to the current interpreter.
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(
full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' %
full_path_or_basename)
return interpreter
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed('Resolving interpreters', V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
if options.rc_file or not ENV.PEX_IGNORE_RCFILES:
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', None)
else:
pex_python_path = None
interpreters = list(
iter_compatible_interpreters(pex_python_path, constraints))
if not interpreters:
die('Could not find compatible interpreter',
CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters) if interpreters else None
pex_builder = PEXBuilder(path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
# NB: `None` means use the default (pypi) index, `[]` means use no indexes.
indexes = None
if options.indexes != [_PYPI] and options.indexes is not None:
indexes = [str(index) for index in options.indexes]
with TRACER.timed(
'Resolving distributions ({})'.format(reqs +
options.requirement_files)):
try:
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=options.platforms,
indexes=indexes,
find_links=options.find_links,
cache=options.cache_dir,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors)
for resolved_dist in resolveds:
log(' %s -> %s' %
(resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def install_distributions(self,
ignore_errors=False,
workspace=None,
max_parallel_jobs=None):
if not any((self._build_requests, self._install_requests)):
# Nothing to build or install.
return []
cache = self._cache or workspace or safe_mkdtemp()
built_wheels_dir = os.path.join(cache, 'built_wheels')
spawn_wheel_build = functools.partial(self._spawn_wheel_build,
built_wheels_dir)
installed_wheels_dir = os.path.join(cache, PexInfo.INSTALL_CACHE)
spawn_install = functools.partial(self._spawn_install,
installed_wheels_dir)
to_install = self._install_requests[:]
to_calculate_requirements_for = []
# 1. Build local projects and sdists.
if self._build_requests:
with TRACER.timed('Building distributions for:'
'\n {}'.format('\n '.join(
map(str, self._build_requests)))):
build_requests, install_requests = self._categorize_build_requests(
build_requests=self._build_requests,
dist_root=built_wheels_dir)
to_install.extend(install_requests)
for build_result in execute_parallel(
inputs=build_requests,
spawn_func=spawn_wheel_build,
raise_type=Untranslateable,
max_jobs=max_parallel_jobs):
to_install.extend(build_result.finalize_build())
# 2. Install wheels in individual chroots.
# Dedup by wheel name; e.g.: only install universal wheels once even though they'll get
# downloaded / built for each interpreter or platform.
install_requests_by_wheel_file = OrderedDict()
for install_request in to_install:
install_requests = install_requests_by_wheel_file.setdefault(
install_request.wheel_file, [])
install_requests.append(install_request)
representative_install_requests = [
requests[0]
for requests in install_requests_by_wheel_file.values()
]
def add_requirements_requests(install_result):
install_requests = install_requests_by_wheel_file[
install_result.request.wheel_file]
to_calculate_requirements_for.extend(
install_result.finalize_install(install_requests))
with TRACER.timed('Installing:'
'\n {}'.format('\n '.join(
map(str, representative_install_requests)))):
install_requests, install_results = self._categorize_install_requests(
install_requests=representative_install_requests,
installed_wheels_dir=installed_wheels_dir)
for install_result in install_results:
add_requirements_requests(install_result)
for install_result in execute_parallel(inputs=install_requests,
spawn_func=spawn_install,
raise_type=Untranslateable,
max_jobs=max_parallel_jobs):
add_requirements_requests(install_result)
# 3. Calculate the final installed requirements.
with TRACER.timed('Calculating installed requirements for:'
'\n {}'.format('\n '.join(
map(str, to_calculate_requirements_for)))):
distribution_requirements = DistributionRequirements.merged(
execute_parallel(inputs=to_calculate_requirements_for,
spawn_func=DistributionRequirements.Request.
spawn_calculation,
raise_type=Untranslateable,
max_jobs=max_parallel_jobs))
installed_distributions = OrderedSet()
for requirements_request in to_calculate_requirements_for:
for distribution in requirements_request.distributions:
installed_distributions.add(
InstalledDistribution(
target=requirements_request.target,
requirement=distribution_requirements.to_requirement(
distribution),
distribution=distribution))
if not ignore_errors:
self._check_install(installed_distributions)
return installed_distributions
def resolve(self, reqs):
# type: (Iterable[Requirement]) -> Iterable[Distribution]
self._update_candidate_distributions(self._load_internal_cache())
unresolved_reqs = OrderedDict(
) # type: OrderedDict[Requirement, OrderedSet]
def record_unresolved(dist_not_found):
# type: (_DistributionNotFound) -> None
TRACER.log("Failed to resolve a requirement: {}".format(
dist_not_found.requirement))
requirers = unresolved_reqs.get(dist_not_found.requirement)
if requirers is None:
requirers = OrderedSet()
unresolved_reqs[dist_not_found.requirement] = requirers
if dist_not_found.required_by:
requirers.add(dist_not_found.required_by)
resolved_dists_by_key = OrderedDict(
) # type: OrderedDict[_RequirementKey, Distribution]
for qualified_req_or_not_found in self._root_requirements_iter(reqs):
if isinstance(qualified_req_or_not_found, _DistributionNotFound):
record_unresolved(qualified_req_or_not_found)
continue
with TRACER.timed("Resolving {}".format(
qualified_req_or_not_found.requirement),
V=2):
for not_found in self._resolve_requirement(
requirement=qualified_req_or_not_found.requirement,
required=qualified_req_or_not_found.required,
resolved_dists_by_key=resolved_dists_by_key,
):
record_unresolved(not_found)
if unresolved_reqs:
TRACER.log("Unresolved requirements:")
for req in unresolved_reqs:
TRACER.log(" - {}".format(req))
TRACER.log("Distributions contained within this pex:")
if not self._pex_info.distributions:
TRACER.log(" None")
else:
for dist_name in self._pex_info.distributions:
TRACER.log(" - {}".format(dist_name))
if not self._pex_info.ignore_errors:
items = []
for index, (requirement,
requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ""
if requirers:
rendered_requirers = "\n Required by:" "\n {requirers}".format(
requirers="\n ".join(map(str, requirers)))
contains = self._available_ranked_dists_by_key[
requirement.key]
if contains:
rendered_contains = (
"\n But this pex only contains:"
"\n {distributions}".format(
distributions="\n ".join(
os.path.basename(
ranked_dist.distribution.location)
for ranked_dist in contains), ))
else:
rendered_contains = (
"\n But this pex had no {project_name!r} distributions."
.format(project_name=requirement.project_name))
items.append("{index: 2d}: {requirement}"
"{rendered_requirers}"
"{rendered_contains}".format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
rendered_contains=rendered_contains,
))
raise ResolveError(
"Failed to resolve requirements from PEX environment @ {pex}.\n"
"Needed {platform} compatible dependencies for:\n"
"{items}".format(pex=self._pex,
platform=self._platform,
items="\n".join(items)))
return OrderedSet(resolved_dists_by_key.values())
def maybe_reexec_pex(compatibility_constraints=None):
"""Handle environment overrides for the Python interpreter to use when executing this pex.
This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO
metadata. If PEX_PYTHON is set it attempts to obtain the binary location of the interpreter
specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for a matching
interpreter in accordance with the interpreter constraints. If both variables are present, this
function gives precedence to PEX_PYTHON_PATH and errors out if no compatible interpreters can be
found on said path.
If neither variable is set, we fall back to plain PEX execution using PATH searching or the
currently executing interpreter. If compatibility constraints are used, we match those constraints
against these interpreters.
:param compatibility_constraints: optional list of requirements-style strings that constrain the
Python interpreter to re-exec this pex with.
"""
current_interpreter = PythonInterpreter.get()
# NB: Used only for tests.
if '_PEX_EXEC_CHAIN' in os.environ:
flag_or_chain = os.environ.pop('_PEX_EXEC_CHAIN')
pex_exec_chain = [] if flag_or_chain == '1' else flag_or_chain.split(
os.pathsep)
pex_exec_chain.append(current_interpreter.binary)
os.environ['_PEX_EXEC_CHAIN'] = os.pathsep.join(pex_exec_chain)
current_interpreter_blessed_env_var = '_PEX_SHOULD_EXIT_BOOTSTRAP_REEXEC'
if os.environ.pop(current_interpreter_blessed_env_var, None):
# We've already been here and selected an interpreter. Continue to execution.
return
from . import pex
pythonpath = pex.PEX.stash_pythonpath()
if pythonpath is not None:
TRACER.log('Stashed PYTHONPATH of {}'.format(pythonpath), V=2)
with TRACER.timed('Selecting runtime interpreter', V=3):
if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH:
# preserve PEX_PYTHON re-exec for backwards compatibility
# TODO: Kill this off completely in favor of PEX_PYTHON_PATH
# https://github.com/pantsbuild/pex/issues/431
TRACER.log('Using PEX_PYTHON={} constrained by {}'.format(
ENV.PEX_PYTHON, compatibility_constraints),
V=3)
target = _select_pex_python_interpreter(
pex_python=ENV.PEX_PYTHON,
compatibility_constraints=compatibility_constraints)
elif ENV.PEX_PYTHON_PATH or compatibility_constraints:
TRACER.log('Using {path} constrained by {constraints}'.format(
path='PEX_PYTHON_PATH={}'.format(ENV.PEX_PYTHON_PATH)
if ENV.PEX_PYTHON_PATH else '$PATH',
constraints=compatibility_constraints),
V=3)
target = _select_path_interpreter(
path=ENV.PEX_PYTHON_PATH,
compatibility_constraints=compatibility_constraints)
elif pythonpath is None:
TRACER.log(
'Using the current interpreter {} since no constraints have been specified and '
'PYTHONPATH is not set.'.format(sys.executable),
V=3)
return
else:
target = current_interpreter
os.environ.pop('PEX_PYTHON', None)
os.environ.pop('PEX_PYTHON_PATH', None)
if pythonpath is None and target == current_interpreter:
TRACER.log(
'Using the current interpreter {} since it matches constraints and '
'PYTHONPATH is not set.'.format(sys.executable))
return
target_binary = target.binary
cmdline = [target_binary] + sys.argv
TRACER.log('Re-executing: '
'cmdline={cmdline!r}, '
'sys.executable={python!r}, '
'PEX_PYTHON={pex_python!r}, '
'PEX_PYTHON_PATH={pex_python_path!r}, '
'COMPATIBILITY_CONSTRAINTS={compatibility_constraints!r}'
'{pythonpath}"'.format(
cmdline=' '.join(cmdline),
python=sys.executable,
pex_python=ENV.PEX_PYTHON,
pex_python_path=ENV.PEX_PYTHON_PATH,
compatibility_constraints=compatibility_constraints,
pythonpath=', (stashed) PYTHONPATH="{}"'.format(pythonpath)
if pythonpath is not None else ''))
# Avoid a re-run through compatibility_constraint checking.
os.environ[current_interpreter_blessed_env_var] = '1'
os.execv(target_binary, cmdline)
def _update_candidate_distributions(self, distribution_iter):
for dist in distribution_iter:
if self.can_add(dist):
with TRACER.timed("Adding %s" % dist, V=2):
self.add(dist)
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
interpreters = [
get_interpreter(interpreter,
options.interpreter_cache_dir,
options.repos,
options.use_wheel)
for interpreter in options.python or [None]
]
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = _lowest_version_interpreter(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt, resolver_option_builder):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platform,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed)
for dist in resolveds:
log(' %s' % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir,
CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' %
(sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(
sdist,
os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
