def test_resolvables_from_iterable():
builder = ResolverOptionsBuilder()
reqs = [
'foo', # string
Package.from_href('foo-2.3.4.tar.gz'), # Package
pkg_resources.Requirement.parse('foo==2.3.4'),
]
resolved_reqs = list(resolvables_from_iterable(reqs, builder))
assert resolved_reqs == [
ResolvableRequirement.from_string('foo', builder),
ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder),
ResolvableRequirement.from_string('foo==2.3.4', builder),
]
def test_resolvables_from_iterable():
builder = ResolverOptionsBuilder()
reqs = [
'foo', # string
Package.from_href('foo-2.3.4.tar.gz'), # Package
pkg_resources.Requirement.parse('foo==2.3.4'),
]
resolved_reqs = list(resolvables_from_iterable(reqs, builder))
assert resolved_reqs == [
ResolvableRequirement.from_string('foo', builder),
ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder),
ResolvableRequirement.from_string('foo==2.3.4', builder),
]
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]
if options.interpreter_constraint:
# NB: options.python and interpreter constraints cannot be used together, so this will not
# affect usages of the interpreter(s) specified by the "--python" command line flag.
constraints = options.interpreter_constraint
validate_constraints(constraints)
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
interpreters = find_compatible_interpreters(pex_python_path, constraints)
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt,
builder=resolver_option_builder,
interpreter=interpreter))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
builder=resolver_option_builder,
interpreter=interpreter):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platforms,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def resolve(requirements,
fetchers=None,
interpreter=None,
platform=None,
context=None,
precedence=None,
cache=None,
cache_ttl=None,
allow_prereleases=None,
use_manylinux=None,
transitive=True):
"""Produce all distributions needed to (recursively) meet `requirements`
:param requirements: An iterator of Requirement-like things, either
:class:`pkg_resources.Requirement` objects or requirement strings.
:keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If
unspecified, the default is to look for packages on PyPI.
:keyword interpreter: (optional) A :class:`PythonInterpreter` object to use for building
distributions and for testing distribution compatibility.
:keyword versions: (optional) a list of string versions, of the form ["33", "32"],
or None. The first version will be assumed to support our ABI.
:keyword platform: (optional) specify the exact platform you want valid
tags for, or None. If None, use the local system platform.
:keyword impl: (optional) specify the exact implementation you want valid
tags for, or None. If None, use the local interpreter impl.
:keyword abi: (optional) specify the exact abi you want valid
tags for, or None. If None, use the local interpreter abi.
:keyword context: (optional) A :class:`Context` object to use for network access. If
unspecified, the resolver will attempt to use the best available network context.
:keyword precedence: (optional) An ordered list of allowable :class:`Package` classes
to be used for producing distributions. For example, if precedence is supplied as
``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and
eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and
eggs will be used, but the resolver will not resort to building anything from source.
:keyword cache: (optional) A directory to use to cache distributions locally.
:keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when
resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is
available in the cache, it will always be used. However, if a non-exact requirement such as
``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than
cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution
is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified,
resolving inexact requirements will always result in making network calls through the
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
:keyword transitive: (optional) Whether or not to resolve transitive dependencies.
:returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
a particular requirement.
This method improves upon the setuptools dependency resolution algorithm by maintaining sets of
all compatible distributions encountered for each requirement rather than the single best
distribution encountered for each requirement. This prevents situations where ``tornado`` and
``tornado==2.0`` could be treated as incompatible with each other because the "best
distribution" when encountering ``tornado`` was tornado 3.0. Instead, ``resolve`` maintains the
set of compatible distributions for each requirement as it is encountered, and iteratively filters
the set. If the set of distributions ever becomes empty, then ``Unsatisfiable`` is raised.
.. versionchanged:: 0.8
A number of keywords were added to make requirement resolution slightly easier to configure.
The optional ``obtainer`` keyword was replaced by ``fetchers``, ``translator``, ``context``,
``threads``, ``precedence``, ``cache`` and ``cache_ttl``, also all optional keywords.
.. versionchanged:: 1.0
The ``translator`` and ``threads`` keywords have been removed. The choice of threading
policy is now implicit. The choice of translation policy is dictated by ``precedence``
directly.
.. versionchanged:: 1.0
``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
classes.
.. versionchanged:: 1.5.0
The ``pkg_blacklist`` has been removed and the return type changed to a list of
:class:`ResolvedDistribution`.
"""
builder = ResolverOptionsBuilder(fetchers=fetchers,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
precedence=precedence,
context=context)
if cache:
resolver = CachingResolver(cache,
cache_ttl,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
interpreter=interpreter,
platform=platform)
else:
resolver = Resolver(allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
interpreter=interpreter,
platform=platform)
return resolver.resolve(
resolvables_from_iterable(requirements,
builder,
interpreter=interpreter))
def build_pex(args, options, resolver_option_builder):
with TRACER.timed('Resolving interpreters', V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]]
if options.interpreter_constraint:
# NB: options.python and interpreter constraints cannot be used together, so this will not
# affect usages of the interpreter(s) specified by the "--python" command line flag.
constraints = options.interpreter_constraint
validate_constraints(constraints)
if options.rc_file or not ENV.PEX_IGNORE_RCFILES:
rc_variables = Variables.from_rc(rc=options.rc_file)
pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
else:
pex_python_path = ""
interpreters = find_compatible_interpreters(pex_python_path, constraints)
if not interpreters:
die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
interpreter = min(interpreters)
pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
fn(src_file_path, dst_path)
for directory in options.sources_directory:
walk_and_do(pex_builder.add_source, directory)
for directory in options.resources_directory:
walk_and_do(pex_builder.add_resource, directory)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = options.inherit_path
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
for requirements_txt in options.requirement_files:
resolvables.extend(requirements_from_file(requirements_txt,
builder=resolver_option_builder,
interpreter=interpreter))
# pip states the constraints format is identical tor requirements
# https://pip.pypa.io/en/stable/user_guide/#constraints-files
for constraints_txt in options.constraint_files:
constraints = []
for r in requirements_from_file(constraints_txt,
builder=resolver_option_builder,
interpreter=interpreter):
r.is_constraint = True
constraints.append(r)
resolvables.extend(constraints)
with TRACER.timed('Resolving distributions'):
try:
resolveds = resolve_multi(resolvables,
interpreters=interpreters,
platforms=options.platforms,
cache=options.cache_dir,
cache_ttl=options.cache_ttl,
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
V=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)
if options.entry_point and options.script:
die('Must specify at most one entry point or script.', INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def resolve(requirements,
fetchers=None,
interpreter=None,
platform=None,
context=None,
precedence=None,
cache=None,
cache_ttl=None,
allow_prereleases=None,
use_manylinux=None):
"""Produce all distributions needed to (recursively) meet `requirements`
:param requirements: An iterator of Requirement-like things, either
:class:`pkg_resources.Requirement` objects or requirement strings.
:keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If
unspecified, the default is to look for packages on PyPI.
:keyword interpreter: (optional) A :class:`PythonInterpreter` object to use for building
distributions and for testing distribution compatibility.
:keyword versions: (optional) a list of string versions, of the form ["33", "32"],
or None. The first version will be assumed to support our ABI.
:keyword platform: (optional) specify the exact platform you want valid
tags for, or None. If None, use the local system platform.
:keyword impl: (optional) specify the exact implementation you want valid
tags for, or None. If None, use the local interpreter impl.
:keyword abi: (optional) specify the exact abi you want valid
tags for, or None. If None, use the local interpreter abi.
:keyword context: (optional) A :class:`Context` object to use for network access. If
unspecified, the resolver will attempt to use the best available network context.
:keyword precedence: (optional) An ordered list of allowable :class:`Package` classes
to be used for producing distributions. For example, if precedence is supplied as
``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and
eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and
eggs will be used, but the resolver will not resort to building anything from source.
:keyword cache: (optional) A directory to use to cache distributions locally.
:keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when
resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is
available in the cache, it will always be used. However, if a non-exact requirement such as
``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than
cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution
is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified,
resolving inexact requirements will always result in making network calls through the
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
:returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
a particular requirement.
This method improves upon the setuptools dependency resolution algorithm by maintaining sets of
all compatible distributions encountered for each requirement rather than the single best
distribution encountered for each requirement. This prevents situations where ``tornado`` and
``tornado==2.0`` could be treated as incompatible with each other because the "best
distribution" when encountering ``tornado`` was tornado 3.0. Instead, ``resolve`` maintains the
set of compatible distributions for each requirement as it is encountered, and iteratively filters
the set. If the set of distributions ever becomes empty, then ``Unsatisfiable`` is raised.
.. versionchanged:: 0.8
A number of keywords were added to make requirement resolution slightly easier to configure.
The optional ``obtainer`` keyword was replaced by ``fetchers``, ``translator``, ``context``,
``threads``, ``precedence``, ``cache`` and ``cache_ttl``, also all optional keywords.
.. versionchanged:: 1.0
The ``translator`` and ``threads`` keywords have been removed. The choice of threading
policy is now implicit. The choice of translation policy is dictated by ``precedence``
directly.
.. versionchanged:: 1.0
``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
classes.
.. versionchanged:: 1.5.0
The ``pkg_blacklist`` has been removed and the return type changed to a list of
:class:`ResolvedDistribution`.
"""
builder = ResolverOptionsBuilder(fetchers=fetchers,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
precedence=precedence,
context=context)
if cache:
resolver = CachingResolver(cache,
cache_ttl,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
interpreter=interpreter,
platform=platform)
else:
resolver = Resolver(allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
interpreter=interpreter,
platform=platform)
return resolver.resolve(resolvables_from_iterable(requirements, builder, interpreter=interpreter))