def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist
# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
distributions = resolve(requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
platform=expand_and_maybe_adjust_platform(
interpreter=interpreter,
# The local interpreter cache is, by definition, composed of
# interpreters for the 'current' platform.
platform='current'),
context=self._python_repos.get_network_context(),
precedence=precedence)
if not distributions:
return None
assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, distributions))))
dist_location = distributions[0].location
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
self._logger(' installed {}'.format(target_location))
return Package.from_href(target_location)
def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist
# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
distributions = resolve(requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
context=self._python_repos.get_network_context(),
precedence=precedence)
if not distributions:
return None
assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, distributions))))
dist_location = distributions[0].location
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
self._logger(' installed {}'.format(target_location))
return Package.from_href(target_location)
def test_deterministic_packages_from_multiple_crawlers():
req = 'protobuf==3.9.1'
options_builder = ResolverOptionsBuilder(precedence=(WheelPackage, ))
resolvable = ResolvableRequirement.from_string(req, options_builder)
pypi_source = Package.from_href(
'https://pypi.org/simple/protobuf/protobuf-3.9.1.tar.gz')
pypi_wheel = Package.from_href(
'https://pypi.org/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl')
internal_wheel = Package.from_href(
'https://packages.internal.example/simple/protobuf/protobuf-3.9.1-py2.py3-none-any.whl'
)
# Multiple fetchers with equally-preferred packages should result in the same package every time,
# regardless the order that crawlers returned them.
mock_iterator = mock.create_autospec(Iterator, spec_set=True)
mock_iterator.iter.return_value = iter(
[pypi_source, pypi_wheel, internal_wheel])
url_order_one = [r.url for r in resolvable.compatible(mock_iterator)]
mock_iterator.iter.return_value = iter(
[pypi_source, internal_wheel, pypi_wheel])
url_order_two = [r.url for r in resolvable.compatible(mock_iterator)]
assert url_order_one == url_order_two
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
def from_string(cls,
requirement_string,
options_builder,
interpreter=None):
requirement_string, extras = strip_extras(requirement_string)
if cls.is_installable(requirement_string):
if interpreter is None:
raise cls.InvalidRequirement(
'%s is not an installable directory because we were called '
'without an interpreter to use to execute setup.py.' %
requirement_string)
packager = Packager(requirement_string, interpreter=interpreter)
try:
sdist = packager.sdist()
except SetuptoolsInstallerBase.Error:
raise cls.InvalidRequirement(
'Could not create source distribution for %s' %
requirement_string)
package = Package.from_href(sdist)
return ResolvablePackage(package,
options_builder.build(package.name),
extras=extras)
else:
raise cls.InvalidRequirement(
'%s does not appear to be an installable directory.' %
requirement_string)
def test_setuptools_version(self):
self.create_file('src/python/foo/__init__.py')
self.create_python_library(
relpath='src/python/foo/commands',
name='commands',
source_contents_map={
'print_sys_path.py':
dedent("""
import os
import sys
from setuptools import Command
class PrintSysPath(Command):
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
with open(os.path.join(os.path.dirname(__file__), 'sys_path.txt'), 'w') as fp:
fp.write(os.linesep.join(sys.path))
""")
},
)
foo = self.create_python_library(relpath='src/python/foo',
name='foo',
dependencies=[
'src/python/foo/commands',
],
provides=dedent("""
setup_py(
name='foo',
version='0.0.0',
)
"""))
self.set_options(run='print_sys_path')
# Make sure setup.py can see our custom distutils Command 'print_sys_path'.
sdist_srcdir = os.path.join(self.distdir, 'foo-0.0.0', 'src')
with environment_as(PYTHONPATH=sdist_srcdir):
with self.run_execute(foo):
with open(
os.path.join(sdist_srcdir, 'foo', 'commands',
'sys_path.txt')) as fp:
# The 1st element of the sys.path should be our custom SetupPyRunner Installer's
# setuptools mixin, which should match the setuptools version specified by the
# PythonSetup subsystem.
package = Package.from_href(fp.readline().strip())
self.assertEqual('setuptools', package.name)
init_subsystem(PythonSetup)
self.assertEqual(
PythonSetup.global_instance().setuptools_version,
package.raw_version)
def from_string(cls,
requirement_string,
options_builder,
interpreter=None):
requirement_string, extras = strip_extras(requirement_string)
package = Package.from_href(requirement_string)
if package is None:
raise cls.InvalidRequirement(
'Requirement string does not appear to be a package.')
return cls(package, options_builder.build(package.name), extras=extras)
def build(self, package, options):
context = options.get_context()
translator = options.get_translator(self._interpreter, self._supported_tags)
with TRACER.timed('Fetching %s' % package.url, V=2):
local_package = Package.from_href(context.fetch(package))
if local_package is None:
raise Untranslateable('Could not fetch package %s' % package)
with TRACER.timed('Translating %s into distribution' % local_package.local_path, V=2):
dist = translator.translate(local_package)
if dist is None:
raise Untranslateable('Package %s is not translateable by %s' % (package, translator))
if not distribution_compatible(dist, self._supported_tags):
raise Untranslateable(
'Could not get distribution for %s on platform %s.' % (package, self._platform))
return dist
def test_resolvables_from_iterable():
builder = ResolverOptionsBuilder()
reqs = [
'foo', # string
Package.from_href('foo-2.3.4.tar.gz'), # Package
pkg_resources.Requirement.parse('foo==2.3.4'),
]
resolved_reqs = list(resolvables_from_iterable(reqs, builder))
assert resolved_reqs == [
ResolvableRequirement.from_string('foo', builder),
ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder),
ResolvableRequirement.from_string('foo==2.3.4', builder),
]
def test_resolvables_from_iterable():
builder = ResolverOptionsBuilder()
reqs = [
'foo', # string
Package.from_href('foo-2.3.4.tar.gz'), # Package
pkg_resources.Requirement.parse('foo==2.3.4'),
]
resolved_reqs = list(resolvables_from_iterable(reqs, builder))
assert resolved_reqs == [
ResolvableRequirement.from_string('foo', builder),
ResolvablePackage.from_string('foo-2.3.4.tar.gz', builder),
ResolvableRequirement.from_string('foo==2.3.4', builder),
]
def build(self, package, options):
# cache package locally
if package.remote:
package = Package.from_href(options.get_context().fetch(package, into=self.__cache))
os.utime(package.local_path, None)
# build into distribution
dist = super(CachingResolver, self).build(package, options)
# if distribution is not in cache, copy
target = os.path.join(self.__cache, os.path.basename(dist.location))
if not os.path.exists(target):
shutil.copyfile(dist.location, target + '~')
os.rename(target + '~', target)
os.utime(target, None)
return DistributionHelper.distribution_from_path(target)
def translate(self, package, into=None):
"""From a SourcePackage, translate to a binary distribution."""
if not isinstance(package, SourcePackage):
return None
if not package.local:
raise ValueError(
'SourceTranslator cannot translate remote packages.')
installer = None
version = self._interpreter.version
unpack_path = Archiver.unpack(package.local_path)
into = into or safe_mkdtemp()
try:
if self._use_2to3 and version >= (3, ):
with TRACER.timed('Translating 2->3 %s' % package.name):
self.run_2to3(unpack_path)
installer = self._installer_impl(unpack_path,
interpreter=self._interpreter)
with TRACER.timed('Packaging %s' % package.name):
try:
dist_path = installer.bdist()
except self._installer_impl.InstallFailure as e:
TRACER.log('Failed to install package at %s: %s' %
(unpack_path, e))
return None
target_path = os.path.join(into, os.path.basename(dist_path))
safe_copy(dist_path, target_path)
target_package = Package.from_href(target_path)
if not target_package:
TRACER.log('Target path %s does not look like a Package.' %
target_path)
return None
if not target_package.compatible(self._supported_tags):
TRACER.log('Target package %s is not compatible with %s' %
(target_package, self._supported_tags))
return None
return DistributionHelper.distribution_from_path(target_path)
except Exception as e:
TRACER.log('Failed to translate %s' % package)
TRACER.log(traceback.format_exc())
finally:
if installer:
installer.cleanup()
if unpack_path:
safe_rmtree(unpack_path)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' % (sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(sdist, os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(
requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def resolve(self, resolvables, resolvable_set=None):
resolvables = [
(resolvable, None) for resolvable in resolvables
if self.is_resolvable_in_target_interpreter_env(resolvable)
]
resolvable_set = resolvable_set or _ResolvableSet()
processed_resolvables = set()
processed_packages = {}
distributions = {}
while resolvables:
while resolvables:
resolvable, parent = resolvables.pop(0)
if resolvable in processed_resolvables:
continue
packages = self.package_iterator(resolvable,
existing=resolvable_set.get(
resolvable.name))
resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)
built_packages = {}
for resolvable, packages, parent, constraint_only in resolvable_set.packages(
):
if constraint_only:
continue
assert len(
packages
) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
if resolvable.name in processed_packages:
if package == processed_packages[resolvable.name]:
continue
if package not in distributions:
dist = self.build(package, resolvable.options)
built_package = Package.from_href(dist.location)
built_packages[package] = built_package
distributions[built_package] = dist
package = built_package
distribution = distributions[package]
processed_packages[resolvable.name] = package
new_parent = '%s->%s' % (
parent, resolvable) if parent else str(resolvable)
# We patch packaging.markers.default_environment here so we find optional reqs for the
# platform we're building the PEX for, rather than the one we're on.
if self._transitive:
with patched_packing_env(self._target_interpreter_env):
resolvables.extend(
(ResolvableRequirement(req, resolvable.options),
new_parent)
for req in distribution.requires(
extras=resolvable_set.extras(resolvable.name)))
resolvable_set = resolvable_set.replace_built(built_packages)
# We may have built multiple distributions depending upon if we found transitive dependencies
# for the same package. But ultimately, resolvable_set.packages() contains the correct version
# for all packages. So loop through it and only return the package version in
# resolvable_set.packages() that is found in distributions.
dists = []
# No point in proceeding if distributions is empty
if not distributions:
return dists
for resolvable, packages, parent, constraint_only in resolvable_set.packages(
):
if constraint_only:
continue
assert len(
packages
) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
distribution = distributions[package]
if isinstance(resolvable, ResolvableRequirement):
requirement = resolvable.requirement
else:
requirement = distribution.as_requirement()
requirement.extras = tuple(resolvable.extras())
dists.append(
ResolvedDistribution(requirement=requirement,
distribution=distribution))
return dists
def test_setuptools_version(self):
self.create_file('src/python/foo/__init__.py')
self.create_python_library(
relpath='src/python/foo/commands',
name='commands',
source_contents_map={
'print_sys_path.py':
dedent("""
import os
import sys
from setuptools import Command
class PrintSysPath(Command):
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
with open(os.path.join(os.path.dirname(__file__), 'sys_path.txt'), 'w') as fp:
fp.write(os.linesep.join(sys.path))
""")
},
)
foo = self.create_python_library(relpath='src/python/foo',
name='foo',
dependencies=[
'src/python/foo/commands',
],
provides=dedent("""
setup_py(
name='foo',
version='0.0.0',
)
"""))
self.set_options(run='print_sys_path')
# Make sure setup.py can see our custom distutils Command 'print_sys_path'.
sdist_srcdir = os.path.join(self.distdir, 'foo-0.0.0', 'src')
with environment_as(PYTHONPATH=sdist_srcdir):
with self.run_execute(foo):
with open(
os.path.join(sdist_srcdir, 'foo', 'commands',
'sys_path.txt'), 'r') as fp:
load_package = lambda: Package.from_href(fp.readline().
strip())
# We don't care about the ordering of `wheel` and `setuptools` on the `sys.path`, just
# that they are 1st as a group.
extras = {
p.name: p
for p in (load_package(), load_package())
}
def assert_extra(name, expected_version):
package = extras.get(name)
self.assertIsNotNone(package)
self.assertEqual(expected_version, package.raw_version)
# The 1st two elements of the sys.path should be our custom SetupPyRunner Installer's
# setuptools and wheel mixins, which should match the setuptools and wheel versions
# specified by the PythonSetup subsystem.
init_subsystem(PythonSetup)
python_setup = PythonSetup.global_instance()
assert_extra('setuptools', python_setup.setuptools_version)
assert_extra('wheel', python_setup.wheel_version)
def assert_extra(name, expected_version):
package = Package.from_href(fp.readline().strip())
self.assertEqual(name, package.name)
self.assertEqual(expected_version, package.raw_version)
def assert_egg_extra(interpreter, name, version):
location = interpreter.get_location('{}=={}'.format(name, version))
self.assertIsNotNone(location)
self.assertIsInstance(Package.from_href(location), EggPackage)
def assert_extra(name, expected_version): package = Package.from_href(fp.readline().strip()) self.assertEqual(name, package.name) self.assertEqual(expected_version, package.raw_version)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir,
CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' %
(sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(
sdist,
os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def assert_egg_extra(interpreter, name, version):
location = interpreter.get_location('{}=={}'.format(
name, version))
self.assertIsNotNone(location)
self.assertIsInstance(Package.from_href(location), EggPackage)
def resolve(self, resolvables, resolvable_set=None):
resolvables = [(resolvable, None) for resolvable in resolvables
if self.is_resolvable_in_target_interpreter_env(resolvable)]
resolvable_set = resolvable_set or _ResolvableSet()
processed_resolvables = set()
processed_packages = {}
distributions = {}
while resolvables:
while resolvables:
resolvable, parent = resolvables.pop(0)
if resolvable in processed_resolvables:
continue
packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)
built_packages = {}
for resolvable, packages, parent, constraint_only in resolvable_set.packages():
if constraint_only:
continue
assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
if resolvable.name in processed_packages:
if package == processed_packages[resolvable.name]:
continue
if package not in distributions:
dist = self.build(package, resolvable.options)
built_package = Package.from_href(dist.location)
built_packages[package] = built_package
distributions[built_package] = dist
package = built_package
distribution = distributions[package]
processed_packages[resolvable.name] = package
new_parent = '%s->%s' % (parent, resolvable) if parent else str(resolvable)
# We patch packaging.markers.default_environment here so we find optional reqs for the
# platform we're building the PEX for, rather than the one we're on.
with patched_packing_env(self._target_interpreter_env):
resolvables.extend(
(ResolvableRequirement(req, resolvable.options), new_parent) for req in
distribution.requires(extras=resolvable_set.extras(resolvable.name))
)
resolvable_set = resolvable_set.replace_built(built_packages)
# We may have built multiple distributions depending upon if we found transitive dependencies
# for the same package. But ultimately, resolvable_set.packages() contains the correct version
# for all packages. So loop through it and only return the package version in
# resolvable_set.packages() that is found in distributions.
dists = []
# No point in proceeding if distributions is empty
if not distributions:
return dists
for resolvable, packages, parent, constraint_only in resolvable_set.packages():
if constraint_only:
continue
assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
distribution = distributions[package]
if isinstance(resolvable, ResolvableRequirement):
requirement = resolvable.requirement
else:
requirement = distribution.as_requirement()
dists.append(ResolvedDistribution(requirement=requirement,
distribution=distribution))
return dists