def test_resolvables_from_iterable(): builder = ResolverOptionsBuilder() reqs = [ 'foo', # string Package.from_href('foo-2.3.4.tar.gz'), # Package pkg_resources.Requirement.parse('foo==2.3.4'), ] resolved_reqs = list(resolvables_from_iterable(reqs, builder)) assert resolved_reqs == [ ResolvableRequirement.from_string('foo', builder), ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder), ResolvableRequirement.from_string('foo==2.3.4', builder), ]
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def resolve(requirements, fetchers=None, interpreter=None, platform=None, context=None, precedence=None, cache=None, cache_ttl=None, allow_prereleases=None, use_manylinux=None, transitive=True): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either :class:`pkg_resources.Requirement` objects or requirement strings. :keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If unspecified, the default is to look for packages on PyPI. :keyword interpreter: (optional) A :class:`PythonInterpreter` object to use for building distributions and for testing distribution compatibility. :keyword versions: (optional) a list of string versions, of the form ["33", "32"], or None. The first version will be assumed to support our ABI. :keyword platform: (optional) specify the exact platform you want valid tags for, or None. If None, use the local system platform. :keyword impl: (optional) specify the exact implementation you want valid tags for, or None. If None, use the local interpreter impl. :keyword abi: (optional) specify the exact abi you want valid tags for, or None. If None, use the local interpreter abi. :keyword context: (optional) A :class:`Context` object to use for network access. If unspecified, the resolver will attempt to use the best available network context. :keyword precedence: (optional) An ordered list of allowable :class:`Package` classes to be used for producing distributions. For example, if precedence is supplied as ``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and eggs will be used, but the resolver will not resort to building anything from source. :keyword cache: (optional) A directory to use to cache distributions locally. :keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is available in the cache, it will always be used. However, if a non-exact requirement such as ``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified, resolving inexact requirements will always result in making network calls through the ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves. :keyword transitive: (optional) Whether or not to resolve transitive dependencies. :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. This method improves upon the setuptools dependency resolution algorithm by maintaining sets of all compatible distributions encountered for each requirement rather than the single best distribution encountered for each requirement. This prevents situations where ``tornado`` and ``tornado==2.0`` could be treated as incompatible with each other because the "best distribution" when encountering ``tornado`` was tornado 3.0. Instead, ``resolve`` maintains the set of compatible distributions for each requirement as it is encountered, and iteratively filters the set. If the set of distributions ever becomes empty, then ``Unsatisfiable`` is raised. .. versionchanged:: 0.8 A number of keywords were added to make requirement resolution slightly easier to configure. The optional ``obtainer`` keyword was replaced by ``fetchers``, ``translator``, ``context``, ``threads``, ``precedence``, ``cache`` and ``cache_ttl``, also all optional keywords. .. versionchanged:: 1.0 The ``translator`` and ``threads`` keywords have been removed. The choice of threading policy is now implicit. The choice of translation policy is dictated by ``precedence`` directly. .. versionchanged:: 1.0 ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver` classes. .. versionchanged:: 1.5.0 The ``pkg_blacklist`` has been removed and the return type changed to a list of :class:`ResolvedDistribution`. """ builder = ResolverOptionsBuilder(fetchers=fetchers, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, transitive=transitive, precedence=precedence, context=context) if cache: resolver = CachingResolver(cache, cache_ttl, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, transitive=transitive, interpreter=interpreter, platform=platform) else: resolver = Resolver(allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, transitive=transitive, interpreter=interpreter, platform=platform) return resolver.resolve( resolvables_from_iterable(requirements, builder, interpreter=interpreter))
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') else: pex_python_path = "" interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def resolve(requirements, fetchers=None, interpreter=None, platform=None, context=None, precedence=None, cache=None, cache_ttl=None, allow_prereleases=None, use_manylinux=None): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either :class:`pkg_resources.Requirement` objects or requirement strings. :keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If unspecified, the default is to look for packages on PyPI. :keyword interpreter: (optional) A :class:`PythonInterpreter` object to use for building distributions and for testing distribution compatibility. :keyword versions: (optional) a list of string versions, of the form ["33", "32"], or None. The first version will be assumed to support our ABI. :keyword platform: (optional) specify the exact platform you want valid tags for, or None. If None, use the local system platform. :keyword impl: (optional) specify the exact implementation you want valid tags for, or None. If None, use the local interpreter impl. :keyword abi: (optional) specify the exact abi you want valid tags for, or None. If None, use the local interpreter abi. :keyword context: (optional) A :class:`Context` object to use for network access. If unspecified, the resolver will attempt to use the best available network context. :keyword precedence: (optional) An ordered list of allowable :class:`Package` classes to be used for producing distributions. For example, if precedence is supplied as ``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and eggs will be used, but the resolver will not resort to building anything from source. :keyword cache: (optional) A directory to use to cache distributions locally. :keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is available in the cache, it will always be used. However, if a non-exact requirement such as ``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified, resolving inexact requirements will always result in making network calls through the ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves. :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. This method improves upon the setuptools dependency resolution algorithm by maintaining sets of all compatible distributions encountered for each requirement rather than the single best distribution encountered for each requirement. This prevents situations where ``tornado`` and ``tornado==2.0`` could be treated as incompatible with each other because the "best distribution" when encountering ``tornado`` was tornado 3.0. Instead, ``resolve`` maintains the set of compatible distributions for each requirement as it is encountered, and iteratively filters the set. If the set of distributions ever becomes empty, then ``Unsatisfiable`` is raised. .. versionchanged:: 0.8 A number of keywords were added to make requirement resolution slightly easier to configure. The optional ``obtainer`` keyword was replaced by ``fetchers``, ``translator``, ``context``, ``threads``, ``precedence``, ``cache`` and ``cache_ttl``, also all optional keywords. .. versionchanged:: 1.0 The ``translator`` and ``threads`` keywords have been removed. The choice of threading policy is now implicit. The choice of translation policy is dictated by ``precedence`` directly. .. versionchanged:: 1.0 ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver` classes. .. versionchanged:: 1.5.0 The ``pkg_blacklist`` has been removed and the return type changed to a list of :class:`ResolvedDistribution`. """ builder = ResolverOptionsBuilder(fetchers=fetchers, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, precedence=precedence, context=context) if cache: resolver = CachingResolver(cache, cache_ttl, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, platform=platform) else: resolver = Resolver(allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, platform=platform) return resolver.resolve(resolvables_from_iterable(requirements, builder, interpreter=interpreter))