def test_resolver_blacklist():
if PY2:
blacklist = {'project2': '<3'}
required_project = "project2;python_version>'3'"
else:
blacklist = {'project2': '>3'}
required_project = "project2;python_version<'3'"
project1 = make_sdist(name='project1',
version='1.0.0',
install_reqs=[required_project])
project2 = make_sdist(name='project2', version='1.1.0')
with temporary_dir() as td:
safe_copy(project1, os.path.join(td, os.path.basename(project1)))
safe_copy(project2, os.path.join(td, os.path.basename(project2)))
fetchers = [Fetcher([td])]
dists = resolve(['project1'], fetchers=fetchers)
assert len(dists) == 2
dists = resolve(['project1'],
fetchers=fetchers,
pkg_blacklist=blacklist)
assert len(dists) == 1
def test_pex_run_conflicting_custom_setuptools_useable():
# Here we use an older setuptools to build the pex which has a newer setuptools requirement.
# These setuptools dists have different pkg_resources APIs:
# $ diff \
# <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \
# <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort)
# 2a3,4
# > pkg_resources/py31compat.py
# > pkg_resources/_vendor/appdirs.py
with temporary_dir() as resolve_cache:
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)]
interpreter = PythonInterpreter.from_binary(sys.executable,
path_extras=[dist.location for dist in dists],
include_site_extras=False)
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from pkg_resources import appdirs, py31compat',
dists=dists,
interpreter=interpreter
)
rc = PEX(pex.path()).run()
assert rc == 0
def test_empty_resolve():
empty_resolve = resolve([])
assert empty_resolve == []
with temporary_dir() as td:
empty_resolve = resolve([], cache=td)
assert empty_resolve == []
def test_thats_it_thats_the_test():
empty_resolve = resolve([])
assert empty_resolve == set()
with temporary_dir() as td:
empty_resolve = resolve([], cache=td)
assert empty_resolve == set()
def test_thats_it_thats_the_test():
empty_resolve = resolve([])
assert empty_resolve == set()
with temporary_dir() as td:
empty_resolve = resolve([], cache=td)
assert empty_resolve == set()
def test_empty_resolve():
empty_resolve = resolve([])
assert empty_resolve == []
with temporary_dir() as td:
empty_resolve = resolve([], cache=td)
assert empty_resolve == []
def test_cached_dependency_pinned_unpinned_resolution_multi_run():
# This exercises the issue described here: https://github.com/pantsbuild/pex/issues/178
project1_0_0 = make_sdist(name='project', version='1.0.0')
project1_1_0 = make_sdist(name='project', version='1.1.0')
with temporary_dir() as td:
for sdist in (project1_0_0, project1_1_0):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
# First run, pinning 1.0.0 in the cache
dists = resolve(['project', 'project==1.0.0'], fetchers=fetchers, cache=cd, cache_ttl=1000)
assert len(dists) == 1
assert dists[0].version == '1.0.0'
# This simulates separate invocations of pex but allows us to keep the same tmp cache dir
Crawler.reset_cache()
# Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also
# return SourcePackages found in td
dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1000)
assert len(dists) == 1
assert dists[0].version == '1.1.0'
# Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact
# resolvable should pull from pypi as well since inexact will and the resulting
# resolvable_set.merge() would fail.
Crawler.reset_cache()
time.sleep(1)
dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd,
cache_ttl=1)
assert len(dists) == 1
assert dists[0].version == '1.1.0'
def test_osx_platform_intel_issue_523():
def bad_interpreter(include_site_extras=True):
return PythonInterpreter.from_binary(
_KNOWN_BAD_APPLE_INTERPRETER,
include_site_extras=include_site_extras)
interpreter = bad_interpreter(include_site_extras=False)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for resolved_dist in resolver.resolve(
[requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version,
dist.location)
with nested(
yield_pex_builder(installer_impl=WheelInstaller,
interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage,
WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
args = [
'-c',
'import pkg_resources; print(pkg_resources.get_supported_platform())'
]
env = os.environ.copy()
env['PEX_VERBOSE'] = '1'
process = pex.run(args=args,
env=env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(
process.returncode, stderr))
# Verify this all worked under the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes(
'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def _resolve_multi(self, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
platforms = self.get_platforms(self._platforms
or self._python_setup.platforms)
fetchers = self._python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
context = self._python_repos.get_network_context()
for platform in platforms:
requirements_cache_dir = os.path.join(
self._python_setup.resolver_cache_dir,
str(self._interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=self._interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=requirements_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl)
return distributions
def _resolve_multi(self, interpreter, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = PythonSetup.global_instance()
python_repos = PythonRepos.global_instance()
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
for platform in python_setup.platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, str(interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=None if platform == "current" else platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
)
return distributions
def _resolve_multi(self, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a map of platform name -> list
of :class:`pkg_resources.Distribution` instances needed to satisfy them on that platform.
That may involve distributions for multiple platforms.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
platforms = self.get_platforms(self._platforms or self._python_setup.platforms)
fetchers = self._python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
context = self._python_repos.get_network_context()
for platform in platforms:
requirements_cache_dir = os.path.join(self._python_setup.resolver_cache_dir,
str(self._interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=self._interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=requirements_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl,
allow_prereleases=self._python_setup.resolver_allow_prereleases)
return distributions
def add_requirements(builder, cache):
# type: (PEXBuilder, str) -> None
for resolved_dist in resolve(requirements,
cache=cache,
interpreter=builder.interpreter):
builder.add_requirement(resolved_dist.requirement)
builder.add_distribution(resolved_dist.distribution)
def _resolve_multi(self, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
platforms = self.get_platforms(self._platforms or self._python_setup.platforms)
fetchers = self._python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
context = self._python_repos.get_network_context()
for platform in platforms:
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=self._interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=self._python_setup.resolver_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl)
return distributions
def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(
os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist
# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
distributions = resolve(
requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
context=self._python_repos.get_network_context(),
precedence=precedence)
if not distributions:
return None
assert len(distributions) == 1, (
'Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, distributions))))
dist_location = distributions[0].location
target_location = os.path.join(os.path.dirname(target_link),
os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
self._logger(' installed {}'.format(target_location))
return Package.from_href(target_location)
def test_osx_platform_intel_issue_523():
def bad_interpreter():
return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
# setuptools and wheel extras.
with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=pb.interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
def run(args, **env):
pex_env = os.environ.copy()
pex_env['PEX_VERBOSE'] = '1'
pex_env.update(**env)
process = pex.run(args=args,
env=pex_env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
return process.returncode, stdout, stderr
returncode, _, stderr = run(['-c', 'import psutil'])
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
returncode, stdout, stderr = run(['-c', 'import pkg_resources'])
assert 0 != returncode, (
'Isolated pex process succeeded but should not have found pkg-resources:\n'
'STDOUT:\n'
'{}\n'
'STDERR:\n'
'{}'
.format(stdout, stdout, stderr)
)
returncode, stdout, stderr = run(
['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'],
# Let the bad interpreter site-packages setuptools leak in.
PEX_INHERIT_PATH='1'
)
assert 0 == returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
)
# Verify this worked along side the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist
# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
distributions = resolve(requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
context=self._python_repos.get_network_context(),
precedence=precedence)
if not distributions:
return None
assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, distributions))))
dist_location = distributions[0].location
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
self._logger(' installed {}'.format(target_location))
return Package.from_href(target_location)
def test_pex_run_conflicting_custom_setuptools_useable():
# Here we use our vendored, newer setuptools to build the pex which has an older setuptools
# requirement. These setuptools dists have different pkg_resources APIs:
# $ diff \
# <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \
# <(zipinfo -1 setuptools-40.6.2-py2.py3-none-any.whl | grep pkg_resources/ | sort)
# 2a3,4
# > pkg_resources/py31compat.py
# > pkg_resources/_vendor/appdirs.py
resolved_dists = resolve(['setuptools==20.3.1'])
dists = [resolved_dist.distribution for resolved_dist in resolved_dists]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
exe_contents=textwrap.dedent("""
import sys
import pkg_resources
try:
from pkg_resources import appdirs
sys.exit(1)
except ImportError:
pass
try:
from pkg_resources import py31compat
sys.exit(2)
except ImportError:
pass
"""),
dists=dists,
)
rc = PEX(pex.path()).run(env={'PEX_VERBOSE': '9'})
assert rc == 0
def _resolve_multi(self, interpreter, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = PythonSetup.global_instance()
python_repos = PythonRepos.global_instance()
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
for platform in python_setup.platforms:
requirements_cache_dir = os.path.join(
python_setup.resolver_cache_dir, str(interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=None if platform == 'current' else platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl)
return distributions
def test_pex_run_custom_pex_useable():
old_pex_version = '0.7.0'
resolved_dists = resolve(
['pex=={}'.format(old_pex_version), 'setuptools==40.6.3'])
dists = [resolved_dist.distribution for resolved_dist in resolved_dists]
with temporary_dir() as temp_dir:
from pex.version import __version__
pex = write_simple_pex(
temp_dir,
exe_contents=textwrap.dedent("""
import sys
try:
# The 0.7.0 release embedded the version directly in setup.py so it should only be
# available via distribution metadata.
from pex.version import __version__
sys.exit(1)
except ImportError:
import pkg_resources
dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex'))
print(dist.version)
"""),
dists=dists,
)
process = PEX(pex.path()).run(blocking=False, stdout=subprocess.PIPE)
stdout, _ = process.communicate()
assert process.returncode == 0
assert old_pex_version == stdout.strip().decode('utf-8')
assert old_pex_version != __version__
def test_simple_local_resolve():
project_sdist = make_sdist(name='project')
with temporary_dir() as td:
safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist)))
fetchers = [Fetcher([td])]
dists = resolve(['project'], fetchers=fetchers)
assert len(dists) == 1
def _resolve_plugins(self):
logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements)))
return resolver.resolve(self._plugin_requirements,
fetchers=self._python_repos.get_fetchers(),
context=self._python_repos.get_network_context(),
cache=self.plugin_cache_dir,
cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire.
allow_prereleases=PANTS_SEMVER.is_prerelease)
def test_simple_local_resolve():
project_sdist = make_sdist(name='project')
with temporary_dir() as td:
safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist)))
fetchers = [Fetcher([td])]
dists = resolve(['project'], fetchers=fetchers)
assert len(dists) == 1
def _resolve_multi(
self,
interpreter: PythonInterpreter,
requirements: List[PythonRequirement],
platforms: Optional[List[Platform]],
find_links: Optional[List[str]],
) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]:
"""Multi-platform dependency resolution for PEX files.
Returns a tuple containing a list of distributions that must be included in order to satisfy a
set of requirements, and the transitive == requirements for thosee distributions. This may
involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = list(find_links) if find_links else []
find_links.extend(python_repos.repos)
# Individual requirements from pants may have a `repository` link attached to them, which is
# extracted in `self.resolve_distributions()`. When generating a .ipex file with
# `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it
# tries to resolve all the requirements from BOOTSTRAP-PEX-INFO.
self._all_find_links.update(OrderedSet(find_links))
distributions: Dict[str, List[Distribution]] = defaultdict(list)
transitive_requirements: List[PythonRequirement] = []
all_find_links = [*python_repos.repos, *find_links]
for platform in platforms:
requirements_cache_dir = os.path.join(
python_setup.resolver_cache_dir, str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
platform=platform,
indexes=python_repos.indexes,
find_links=all_find_links,
cache=requirements_cache_dir,
allow_prereleases=python_setup.resolver_allow_prereleases,
manylinux=python_setup.manylinux,
)
for resolved_dist in resolved_dists:
dist = resolved_dist.distribution
transitive_requirements.append(dist.as_requirement())
distributions[platform].append(dist)
return (distributions, transitive_requirements)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift',
contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(
one)
self.create_file('src/thrift2/com/foo/two.thrift',
contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base,
synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
pythonpath = [
os.path.join(get_buildroot(), t.target_base) for t in targets
]
for dist in resolve(
['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([
interpreter.binary, '-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'
],
env={
'PYTHONPATH':
os.pathsep.join(pythonpath)
},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _resolve_plugins(self):
logger.info('Resolving new plugins...:\n {}'.format('\n '.join(
self._plugin_requirements)))
return resolver.resolve(
self._plugin_requirements,
fetchers=self._python_repos.get_fetchers(),
context=self._python_repos.get_network_context(),
cache=self.plugin_cache_dir,
cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire.
allow_prereleases=PANTS_SEMVER.is_prerelease)
def assert_resolve(expected_version, **resolve_kwargs):
dists = resolve(
[
'dep>=3.0.0rc1',
'dep==3.0.0rc4',
],
fetchers=fetchers, **resolve_kwargs)
assert 1 == len(dists)
dist = dists[0]
assert expected_version == dist.version
def _resolve_plugins(self):
logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements)))
return resolver.resolve(self._plugin_requirements,
fetchers=self._python_repos.get_fetchers(),
context=self._python_repos.get_network_context(),
cache=self.plugin_cache_dir,
cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire.
allow_prereleases=PANTS_SEMVER.is_prerelease,
# Plugins will all depend on `pantsbuild.pants` which is distributed as
# a manylinux wheel.
use_manylinux=True)
def test_pex_run_custom_setuptools_useable():
resolved_dists = resolve(['setuptools==36.2.7'])
dists = [resolved_dist.distribution for resolved_dist in resolved_dists]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from setuptools.sandbox import run_setup',
dists=dists,
)
rc = PEX(pex.path()).run()
assert rc == 0
def test_pex_run_custom_setuptools_useable():
with temporary_dir() as resolve_cache:
dists = resolve(['setuptools==36.2.7'], cache=resolve_cache)
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from setuptools.sandbox import run_setup',
dists=dists,
)
rc = PEX(pex.path()).run()
assert rc == 0
def test_diamond_local_resolve_cached():
# This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0'])
project2_sdist = make_sdist(name='project2')
with temporary_dir() as dd:
for sdist in (project1_sdist, project2_sdist):
safe_copy(sdist, os.path.join(dd, os.path.basename(sdist)))
fetchers = [Fetcher([dd])]
with temporary_dir() as cd:
dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000)
assert len(dists) == 2
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError(
'Expected interpreter to be a PythonInterpreter, got %s' %
type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache',
default_name='eggs')
platforms = get_platforms(
platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def test_diamond_local_resolve_cached():
# This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0'])
project2_sdist = make_sdist(name='project2')
with temporary_dir() as dd:
for sdist in (project1_sdist, project2_sdist):
safe_copy(sdist, os.path.join(dd, os.path.basename(sdist)))
fetchers = [Fetcher([dd])]
with temporary_dir() as cd:
dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000)
assert len(dists) == 2
def test_resolver_blacklist():
if PY2:
blacklist = {'project2': '<3'}
required_project = "project2;python_version>'3'"
else:
blacklist = {'project2': '>3'}
required_project = "project2;python_version<'3'"
project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project])
project2 = make_sdist(name='project2', version='1.1.0')
with temporary_dir() as td:
safe_copy(project1, os.path.join(td, os.path.basename(project1)))
safe_copy(project2, os.path.join(td, os.path.basename(project2)))
fetchers = [Fetcher([td])]
dists = resolve(['project1'], fetchers=fetchers)
assert len(dists) == 2
dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist)
assert len(dists) == 1
def _resolve_plugins(self):
# When bootstrapping plugins without the full pants python backend machinery in-play, we are not
# guaranteed a properly initialized interpreter with wheel support so we enforce eggs only for
# bdists with this custom precedence.
precedence = (EggPackage, SourcePackage)
logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements)))
return resolver.resolve(self._plugin_requirements,
fetchers=self._python_repos.get_fetchers(),
context=self._python_repos.get_network_context(),
precedence=precedence,
cache=self.plugin_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl)
def _resolve_plugins(self):
logger.info('Resolving new plugins...:\n {}'.format('\n '.join(
self._plugin_requirements)))
return resolver.resolve(
self._plugin_requirements,
fetchers=self._python_repos.get_fetchers(),
context=self._python_repos.get_network_context(),
cache=self.plugin_cache_dir,
cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire.
allow_prereleases=PANTS_SEMVER.is_prerelease,
# Plugins will all depend on `pantsbuild.pants` which is distributed as
# a manylinux wheel.
use_manylinux=True)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = interpreter.extras.values()
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = interpreter.extras.values()
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(
install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
ttl=3600,
find_links=None):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError(
'Expected interpreter to be a PythonInterpreter, got %s' %
type(interpreter))
cache = PythonSetup(config).scratch_dir('install_cache',
default_name='eggs')
platforms = get_platforms(
platforms or config.getlist('python-setup', 'platforms', ['current']))
fetchers = fetchers_from_config(config)
if find_links:
fetchers.extend(Fetcher([path]) for path in find_links)
context = context_from_config(config)
for platform in platforms:
distributions[platform] = resolve(requirements=requirements,
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=cache,
cache_ttl=ttl)
return distributions
def _resolve_plugins(self) -> Iterable[str]:
logger.info(
"Resolving new plugins...:\n {}".format("\n ".join(self._plugin_requirements))
)
resolved_dists = resolver.resolve(
self._plugin_requirements,
indexes=self._python_repos.indexes,
find_links=self._python_repos.repos,
interpreter=self._interpreter,
cache=self.plugin_cache_dir,
allow_prereleases=PANTS_SEMVER.is_prerelease,
)
return [
self._install_plugin(resolved_dist.distribution) for resolved_dist in resolved_dists
]
def resolve_multi(python_setup,
python_repos,
requirements,
interpreter=None,
platforms=None,
ttl=3600,
find_links=None):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param python_setup: Pants :class:`PythonSetup` object.
:param python_repos: Pants :class:`PythonRepos` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got {}'.format(type(interpreter)))
cache = os.path.join(python_setup.scratch_dir, 'eggs')
platforms = get_platforms(platforms or python_setup.platforms)
fetchers = python_repos.get_fetchers()
if find_links:
fetchers.extend(Fetcher([path]) for path in find_links)
context = python_repos.get_network_context()
for platform in platforms:
distributions[platform] = resolve(
requirements=requirements,
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=cache,
cache_ttl=ttl)
return distributions
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
def test_ambiguous_transitive_resolvable():
# If an unbounded or larger bounded resolvable is resolved first, and a
# transitive resolvable is resolved later in another round, Error(Ambiguous resolvable) can be
# raised because foo pulls in foo-2.0.0 and bar->foo==1.0.0 pulls in foo-1.0.0.
foo1_0 = make_sdist(name='foo', version='1.0.0')
foo2_0 = make_sdist(name='foo', version='2.0.0')
bar1_0 = make_sdist(name='bar', version='1.0.0', install_reqs=['foo==1.0.0'])
with temporary_dir() as td:
for sdist in (foo1_0, foo2_0, bar1_0):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
dists = resolve(['foo', 'bar'], fetchers=fetchers, cache=cd,
cache_ttl=1000)
assert len(dists) == 2
assert dists[0].version == '1.0.0'
def test_activate_extras_issue_615():
with yield_pex_builder() as pb:
for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter):
pb.add_requirement(resolved_dist.requirement)
pb.add_dist_location(resolved_dist.distribution.location)
pb.set_script('pex')
pb.freeze()
process = PEX(pb.path(), interpreter=pb.interpreter).run(args=['--version'],
env={'PEX_VERBOSE': '9'},
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr)
)
assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _resolve_multi(self, interpreter, requirements, platforms, find_links):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = find_links or []
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
for platform in platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
allow_prereleases=python_setup.resolver_allow_prereleases,
use_manylinux=python_setup.use_manylinux)
distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
return distributions
