def test_resolvable_requirement():
req = 'foo[bar]==2.3.4'
resolvable = ResolvableRequirement.from_string(
req, ResolverOptionsBuilder(fetchers=[]))
assert resolvable.requirement == pkg_resources.Requirement.parse(
'foo[bar]==2.3.4')
assert resolvable.name == 'foo'
assert resolvable.exact is True
assert resolvable.extras() == ['bar']
assert resolvable.options._fetchers == []
assert resolvable.packages() == []
source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz')
mock_iterator = mock.create_autospec(Iterator, spec_set=True)
mock_iterator.iter.return_value = iter([source_pkg])
assert resolvable.compatible(mock_iterator) == [source_pkg]
assert mock_iterator.iter.mock_calls == [
mock.call(pkg_resources.Requirement.parse('foo[bar]==2.3.4'))
]
# test non-exact
resolvable = ResolvableRequirement.from_string('foo',
ResolverOptionsBuilder())
assert resolvable.exact is False
# test Resolvable.get, which should delegate to a ResolvableRequirement in this case
assert Resolvable.get('foo') == ResolvableRequirement.from_string(
'foo', ResolverOptionsBuilder())
def test_resolvable_package():
builder = ResolverOptionsBuilder()
source_name = 'foo-2.3.4.tar.gz'
pkg = SourcePackage.from_href(source_name)
resolvable = ResolvablePackage.from_string(source_name, builder)
assert resolvable.packages() == [pkg]
mock_iterator = mock.create_autospec(Iterator, spec_set=True)
mock_iterator.iter.return_value = iter([])
# fetchers are currently unused for static packages.
assert resolvable.compatible(mock_iterator) == []
assert mock_iterator.iter.mock_calls == []
assert resolvable.name == 'foo'
assert resolvable.exact is True
assert resolvable.extras() == []
resolvable = ResolvablePackage.from_string(source_name + '[extra1,extra2]',
builder)
assert resolvable.extras() == ['extra1', 'extra2']
assert Resolvable.get('foo-2.3.4.tar.gz') == ResolvablePackage.from_string(
'foo-2.3.4.tar.gz', builder)
with pytest.raises(ResolvablePackage.InvalidRequirement):
ResolvablePackage.from_string('foo', builder)
def pack_in_pex(requirements: Dict[str, str], output: str) -> str:
"""
Pack current environment using a pex.
:param requirements: list of requirements (ex ['tensorflow==0.1.10'])
:param output: location of the pex
:return: destination of the archive, name of the pex
"""
requirements_to_install = format_requirements(requirements)
fetchers = []
if _criteo.is_criteo():
fetchers.append(PyPIFetcher(pypi_base=CRITEO_PYPI_URL))
fetchers.append(PyPIFetcher())
resolver_option_builder = ResolverOptionsBuilder(use_manylinux=True,
fetchers=fetchers)
resolvables = [
Resolvable.get(req, resolver_option_builder)
for req in requirements_to_install
]
pex_builder = PEXBuilder(copy=True)
try:
resolveds = resolve_multi(resolvables, use_manylinux=True)
for resolved in resolveds:
_logger.debug("Add requirement %s", resolved.distribution)
pex_builder.add_distribution(resolved.distribution)
pex_builder.add_requirement(resolved.requirement)
except (Unsatisfiable, Untranslateable):
_logger.exception('Cannot create pex')
raise
pex_builder.build(output)
return output
def test_deterministic_packages_from_multiple_crawlers():
req = 'protobuf==3.9.1'
options_builder = ResolverOptionsBuilder(precedence=(WheelPackage, ))
resolvable = ResolvableRequirement.from_string(req, options_builder)
pypi_source = Package.from_href(
'https://pypi.org/simple/protobuf/protobuf-3.9.1.tar.gz')
pypi_wheel = Package.from_href(
'https://pypi.org/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl')
internal_wheel = Package.from_href(
'https://packages.internal.example/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl'
)
# Multiple fetchers with equally-preferred packages should result in the same package every time,
# regardless the order that crawlers returned them.
mock_iterator = mock.create_autospec(Iterator, spec_set=True)
mock_iterator.iter.return_value = iter(
[pypi_source, pypi_wheel, internal_wheel])
url_order_one = [r.url for r in resolvable.compatible(mock_iterator)]
mock_iterator.iter.return_value = iter(
[pypi_source, internal_wheel, pypi_wheel])
url_order_two = [r.url for r in resolvable.compatible(mock_iterator)]
assert url_order_one == url_order_two
def configure_clp():
usage = (
'%prog [-o OUTPUT.PEX] [options] [-- arg1 arg2 ...]\n\n'
'%prog builds a PEX (Python Executable) file based on the given specifications: '
'sources, requirements, their dependencies and other options.')
parser = OptionParser(usage=usage, version='%prog {0}'.format(__version__))
resolver_options_builder = ResolverOptionsBuilder()
configure_clp_pex_resolution(parser, resolver_options_builder)
configure_clp_pex_options(parser)
configure_clp_pex_environment(parser)
configure_clp_pex_entry_points(parser)
parser.add_option(
'-o',
'--output-file',
dest='pex_name',
default=None,
help='The name of the generated .pex file: Omiting this will run PEX '
'immediately and not save it to a file.')
parser.add_option(
'-r',
'--requirement',
dest='requirement_files',
metavar='FILE',
default=[],
type=str,
action='append',
help=
'Add requirements from the given requirements file. This option can be used multiple '
'times.')
parser.add_option(
'-v',
dest='verbosity',
default=0,
action='callback',
callback=increment_verbosity,
help='Turn on logging verbosity, may be specified multiple times.')
parser.add_option(
'--pex-root',
dest='pex_root',
default=None,
help=
'Specify the pex root used in this invocation of pex. [Default: ~/.pex]'
)
parser.add_option(
'--help-variables',
action='callback',
callback=print_variable_help,
help=
'Print out help about the various environment variables used to change the behavior of '
'a running PEX file.')
return parser, resolver_options_builder
def test_resolvable_set_is_constraint_only():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
c = ResolvableRequirement.from_string('foo', builder)
c.is_constraint = True
package = SourcePackage.from_href('foo-2.3.4.tar.gz')
rs.merge(c, [package])
assert rs.packages() == [(c, set([package]), None, True)]
def test_resolvable_directory():
builder = ResolverOptionsBuilder()
with make_source_dir(name='my_project') as td:
rdir = ResolvableDirectory.from_string(td, builder)
assert rdir.name == pkg_resources.safe_name('my_project')
assert rdir.extras() == []
rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder)
assert rdir.name == pkg_resources.safe_name('my_project')
assert rdir.extras() == ['extra1', 'extra2']
def test_resolvable_set_constraint_and_non_constraint():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
constraint = ResolvableRequirement.from_string('foo', builder)
constraint.is_constraint = True
package = SourcePackage.from_href('foo-2.3.4.tar.gz')
rq = ResolvableRequirement.from_string('foo', builder)
rs.merge(constraint, [package])
rs.merge(rq, [package])
assert rs.packages() == [(rq, set([package]), None, False)]
def test_resolver_with_constraint():
builder = ResolverOptionsBuilder()
r = Resolver()
rs = _ResolvableSet()
constraint = ResolvableRequirement.from_string('foo', builder)
constraint.is_constraint = True
package = SourcePackage.from_href('foo-2.3.4.tar.gz')
rq = ResolvableRequirement.from_string('foo', builder)
rs.merge(constraint, [package])
rs.merge(rq, [package])
assert r.resolve([], resolvable_set=rs) == []
def get(cls, resolvable_string, options_builder=None, interpreter=None):
"""Get a :class:`Resolvable` from a string.
:returns: A :class:`Resolvable` or ``None`` if no implementation was appropriate.
"""
options_builder = options_builder or ResolverOptionsBuilder()
for resolvable_impl in cls._REGISTRY:
try:
return resolvable_impl.from_string(resolvable_string,
options_builder,
interpreter=interpreter)
except cls.InvalidRequirement:
continue
raise cls.InvalidRequirement('Unknown requirement type: %s' % resolvable_string)
def test_constraints_range():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
req = ResolvableRequirement.from_string("foo>0.5", builder)
constraint = ResolvableRequirement.from_string("foo<0.9", builder)
constraint.is_constraint = True
version_packages = []
for version in range(1, 10):
version_string = "foo-0.{0}.tar.gz".format(version)
package = SourcePackage.from_href(version_string)
version_packages.append(package)
rs.merge(req, version_packages[4:])
rs.merge(constraint, version_packages[:8])
assert rs.packages() == [(req, set(version_packages[4:8]), None, False)]
def test_resolvables_from_iterable():
builder = ResolverOptionsBuilder()
reqs = [
'foo', # string
Package.from_href('foo-2.3.4.tar.gz'), # Package
pkg_resources.Requirement.parse('foo==2.3.4'),
]
resolved_reqs = list(resolvables_from_iterable(reqs, builder))
assert resolved_reqs == [
ResolvableRequirement.from_string('foo', builder),
ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder),
ResolvableRequirement.from_string('foo==2.3.4', builder),
]
def test_resolvable_set_built():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
rq = ResolvableRequirement.from_string('foo', builder)
source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz')
binary_pkg = EggPackage.from_href('foo-2.3.4-py3.4.egg')
rs.merge(rq, [source_pkg])
assert rs.get('foo') == set([source_pkg])
assert rs.packages() == [(rq, set([source_pkg]), None, False)]
with pytest.raises(Unsatisfiable):
rs.merge(rq, [binary_pkg])
updated_rs = rs.replace_built({source_pkg: binary_pkg})
updated_rs.merge(rq, [binary_pkg])
assert updated_rs.get('foo') == set([binary_pkg])
assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)]
def test_resolvable_set():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
rq = ResolvableRequirement.from_string('foo[ext]', builder)
source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz')
binary_pkg = EggPackage.from_href('Foo-2.3.4-py3.4.egg')
rs.merge(rq, [source_pkg, binary_pkg])
assert rs.get(source_pkg.name) == set([source_pkg, binary_pkg])
assert rs.get(binary_pkg.name) == set([source_pkg, binary_pkg])
assert rs.packages() == [(rq, set([source_pkg, binary_pkg]), None, False)]
# test methods
assert rs.extras('foo') == set(['ext'])
assert rs.extras('Foo') == set(['ext'])
# test filtering
rs.merge(rq, [source_pkg])
assert rs.get('foo') == set([source_pkg])
assert rs.get('Foo') == set([source_pkg])
with pytest.raises(Unsatisfiable):
rs.merge(rq, [binary_pkg])
def parser_pair():
builder = ResolverOptionsBuilder()
parser = OptionParser()
yield builder, parser
def requirements_from_lines(lines,
builder=None,
relpath=None,
interpreter=None):
relpath = relpath or os.getcwd()
builder = builder.clone() if builder else ResolverOptionsBuilder()
to_resolve = []
for line in lines:
line = line.strip()
if not line or line.startswith('#'):
continue
elif line.startswith('-e '):
raise UnsupportedLine('Editable distributions not supported: %s' %
line)
elif _startswith_any(line, ('-i ', '--index-url')):
builder.set_index(_get_parameter(line))
elif line.startswith('--extra-index-url'):
builder.add_index(_get_parameter(line))
elif _startswith_any(line, ('-f ', '--find-links')):
builder.add_repository(_get_parameter(line))
elif line.startswith('--allow-external'):
builder.allow_external(_get_parameter(line))
elif line.startswith('--allow-all-external'):
builder.allow_all_external()
elif line.startswith('--allow-unverified'):
builder.allow_unverified(_get_parameter(line))
elif line.startswith('--pre'):
builder.allow_prereleases(True)
elif line.startswith('--no-pre'):
builder.allow_prereleases(False)
elif line.startswith('--no-index'):
builder.clear_indices()
elif line.startswith('--no-use-wheel'):
builder.no_use_wheel()
# defer the conversion of strings/files to resolvables until all options defined
# within the current grouping of lines has been processed.
elif _startswith_any(line, ('-r ', '--requirement')):
path = os.path.join(relpath, _get_parameter(line))
to_resolve.append(RequirementsTxtSentinel(path))
else:
to_resolve.append(line)
resolvables = []
for resolvable in to_resolve:
if isinstance(resolvable, RequirementsTxtSentinel):
resolvables.extend(
requirements_from_file(resolvable.filename,
builder=builder,
interpreter=interpreter))
else:
try:
resolvables.append(
Resolvable.get(resolvable,
options_builder=builder,
interpreter=interpreter))
except Resolvable.Error as e:
raise UnsupportedLine('Could not resolve line: %s (%s)' %
(resolvable, e))
return resolvables
def test_resolvable_is_constraint_getter_setter():
builder = ResolverOptionsBuilder()
req = ResolvableRequirement.from_string('foo', builder)
assert req.is_constraint is False
req.is_constraint = True
assert req.is_constraint is True
def rr(req): return ResolvableRequirement.from_string(req, ResolverOptionsBuilder())
def _establish_resolver_options(args):
resolver_options_builder = ResolverOptionsBuilder()
if args.use_pypi:
resolver_options_builder.add_index(PyPIFetcher.PYPI_BASE)
for index in args.indicies:
resolver_options_builder.add_index(index)
for repo in args.repos:
resolver_options_builder.add_repository(repo)
if args.build_source:
resolver_options_builder.allow_builds()
else:
resolver_options_builder.no_allow_builds()
if args.use_wheel:
resolver_options_builder.use_wheel()
else:
resolver_options_builder.no_use_wheel()
return resolver_options_builder
def test_resolvable_repository():
# not yet implemented
with pytest.raises(Resolvable.InvalidRequirement):
ResolvableRepository.from_string('git+http://github.com/wickman/pex',
ResolverOptionsBuilder())
def build_site_packages():
"""Use PEX to resolve dependencies in a virtual environment,
with some customizations to reduce the size of our build.
https://www.pantsbuild.org/pex.html
"""
# Remove flywheel_cli from cache
# If you skip this step, it doesn't automatically update the python code
if os.path.isdir(PEX_BUILD_CACHE_DIR):
for name in os.listdir(PEX_BUILD_CACHE_DIR):
if fnmatch.fnmatch(name, 'flywheel_cli*.whl'):
path = os.path.join(PEX_BUILD_CACHE_DIR, name)
print('Removing {} from cache...'.format(name))
os.remove(path)
# Read ignore list
# See package-ignore.txt, largely we're removing test files and
# Multi-megabyte dicoms from the dicom folder
ignore_patterns = read_ignore_patterns()
# Create resolver
# Loosely based on: https://github.com/pantsbuild/pex/blob/982cb9a988949ffff3348b9bca98ae72a0bf8847/pex/bin/pex.py#L577
resolver_option_builder = ResolverOptionsBuilder()
resolvables = [
Resolvable.get('flywheel-cli=={}'.format(PYTHON_CLI_VERSION),
resolver_option_builder)
]
resolver = CachingResolver(PEX_BUILD_CACHE_DIR, None)
# Effectively we resolve (possibly from cache) The source and all of the dependency packages
# Then create the virtual environment, which contains those files
print('Resolving distributions')
resolved = resolver.resolve(resolvables)
print('Building package lists')
builder = PEXBuilder()
for dist in resolved:
builder.add_distribution(dist)
builder.add_requirement(dist.as_requirement())
# After this point, builder.chroot contains a full list of the files
print('Compiling package')
builder.freeze(bytecode_compile=False)
site_packages_path = os.path.join(BUILD_DIR, 'site-packages.zip')
# Create an uncompressed site-packages.zip and add all of the discovered files
# (Except those that are filtered out)
with open(site_packages_path, 'wb') as f:
added_files = set()
with zipfile.ZipFile(f, 'w') as zf:
for filename in sorted(builder.chroot().files()):
if is_ignored_file(ignore_patterns, filename):
continue
if not filename.startswith('.deps'):
continue
# Determine new path
src_path = os.path.join(builder.chroot().chroot, filename)
dst_path = '/'.join(filename.split('/')[2:])
# Optionally, compile the file
_, ext = os.path.splitext(src_path)
if ext == '.py':
cfile_path = src_path + 'c'
dst_path += 'c'
print('Compiling: {}'.format(dst_path))
py_compile.compile(src_path,
cfile=cfile_path,
dfile=dst_path,
optimize=1)
src_path = cfile_path
if not dst_path in added_files:
zf.write(src_path, dst_path)
added_files.add(dst_path)
return site_packages_path
def resolve(requirements,
fetchers=None,
interpreter=None,
platform=None,
context=None,
precedence=None,
cache=None,
cache_ttl=None,
allow_prereleases=None,
use_manylinux=None,
transitive=True):
"""Produce all distributions needed to (recursively) meet `requirements`
:param requirements: An iterator of Requirement-like things, either
:class:`pkg_resources.Requirement` objects or requirement strings.
:keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If
unspecified, the default is to look for packages on PyPI.
:keyword interpreter: (optional) A :class:`PythonInterpreter` object to use for building
distributions and for testing distribution compatibility.
:keyword versions: (optional) a list of string versions, of the form ["33", "32"],
or None. The first version will be assumed to support our ABI.
:keyword platform: (optional) specify the exact platform you want valid
tags for, or None. If None, use the local system platform.
:keyword impl: (optional) specify the exact implementation you want valid
tags for, or None. If None, use the local interpreter impl.
:keyword abi: (optional) specify the exact abi you want valid
tags for, or None. If None, use the local interpreter abi.
:keyword context: (optional) A :class:`Context` object to use for network access. If
unspecified, the resolver will attempt to use the best available network context.
:keyword precedence: (optional) An ordered list of allowable :class:`Package` classes
to be used for producing distributions. For example, if precedence is supplied as
``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and
eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and
eggs will be used, but the resolver will not resort to building anything from source.
:keyword cache: (optional) A directory to use to cache distributions locally.
:keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when
resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is
available in the cache, it will always be used. However, if a non-exact requirement such as
``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than
cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution
is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified,
resolving inexact requirements will always result in making network calls through the
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
:keyword transitive: (optional) Whether or not to resolve transitive dependencies.
:returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
a particular requirement.
This method improves upon the setuptools dependency resolution algorithm by maintaining sets of
all compatible distributions encountered for each requirement rather than the single best
distribution encountered for each requirement. This prevents situations where ``tornado`` and
``tornado==2.0`` could be treated as incompatible with each other because the "best
distribution" when encountering ``tornado`` was tornado 3.0. Instead, ``resolve`` maintains the
set of compatible distributions for each requirement as it is encountered, and iteratively filters
the set. If the set of distributions ever becomes empty, then ``Unsatisfiable`` is raised.
.. versionchanged:: 0.8
A number of keywords were added to make requirement resolution slightly easier to configure.
The optional ``obtainer`` keyword was replaced by ``fetchers``, ``translator``, ``context``,
``threads``, ``precedence``, ``cache`` and ``cache_ttl``, also all optional keywords.
.. versionchanged:: 1.0
The ``translator`` and ``threads`` keywords have been removed. The choice of threading
policy is now implicit. The choice of translation policy is dictated by ``precedence``
directly.
.. versionchanged:: 1.0
``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
classes.
.. versionchanged:: 1.5.0
The ``pkg_blacklist`` has been removed and the return type changed to a list of
:class:`ResolvedDistribution`.
"""
builder = ResolverOptionsBuilder(fetchers=fetchers,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
precedence=precedence,
context=context)
if cache:
resolver = CachingResolver(cache,
cache_ttl,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
interpreter=interpreter,
platform=platform)
else:
resolver = Resolver(allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
transitive=transitive,
interpreter=interpreter,
platform=platform)
return resolver.resolve(
resolvables_from_iterable(requirements,
builder,
interpreter=interpreter))
def configure_clp():
usage = (
'%prog [-o OUTPUT.PEX] [options] [-- arg1 arg2 ...]\n\n'
'%prog builds a PEX (Python Executable) file based on the given specifications: '
'sources, requirements, their dependencies and other options.')
parser = OptionParser(usage=usage, version='%prog {0}'.format(__version__))
resolver_options_builder = ResolverOptionsBuilder()
configure_clp_pex_resolution(parser, resolver_options_builder)
configure_clp_pex_options(parser)
configure_clp_pex_environment(parser)
configure_clp_pex_entry_points(parser)
parser.add_option(
'-o',
'--output-file',
dest='pex_name',
default=None,
help='The name of the generated .pex file: Omiting this will run PEX '
'immediately and not save it to a file.')
parser.add_option(
'-p',
'--preamble-file',
dest='preamble_file',
metavar='FILE',
default=None,
type=str,
help=
'The name of a file to be included as the preamble for the generated .pex file'
)
parser.add_option(
'-D',
'--sources-directory',
dest='sources_directory',
metavar='DIR',
default=[],
type=str,
action='append',
help='Add sources directory to be packaged into the generated .pex file.'
' This option can be used multiple times.')
parser.add_option(
'-R',
'--resources-directory',
dest='resources_directory',
metavar='DIR',
default=[],
type=str,
action='append',
help=
'Add resources directory to be packaged into the generated .pex file.'
' This option can be used multiple times.')
parser.add_option(
'-r',
'--requirement',
dest='requirement_files',
metavar='FILE',
default=[],
type=str,
action='append',
help=
'Add requirements from the given requirements file. This option can be used multiple '
'times.')
parser.add_option(
'--constraints',
dest='constraint_files',
metavar='FILE',
default=[],
type=str,
action='append',
help=
'Add constraints from the given constraints file. This option can be used multiple '
'times.')
parser.add_option(
'-v',
dest='verbosity',
default=0,
action='callback',
callback=increment_verbosity,
help='Turn on logging verbosity, may be specified multiple times.')
parser.add_option(
'--emit-warnings',
'--no-emit-warnings',
dest='emit_warnings',
action='callback',
callback=parse_bool,
default=True,
help=
'Emit runtime UserWarnings on stderr. If false, only emit them when PEX_VERBOSE is set.'
'Default: emit user warnings to stderr')
parser.add_option(
'--pex-root',
dest='pex_root',
default=None,
help=
'Specify the pex root used in this invocation of pex. [Default: ~/.pex]'
)
parser.add_option(
'--help-variables',
action='callback',
callback=print_variable_help,
help=
'Print out help about the various environment variables used to change the behavior of '
'a running PEX file.')
return parser, resolver_options_builder
