def test_resolver_blacklist(): if PY2: blacklist = {'project2': '<3'} required_project = "project2;python_version>'3'" else: blacklist = {'project2': '>3'} required_project = "project2;python_version<'3'" project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project]) project2 = make_sdist(name='project2', version='1.1.0') with temporary_dir() as td: safe_copy(project1, os.path.join(td, os.path.basename(project1))) safe_copy(project2, os.path.join(td, os.path.basename(project2))) fetchers = [Fetcher([td])] dists = resolve(['project1'], fetchers=fetchers) assert len(dists) == 2 dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist) assert len(dists) == 1
def test_pex_run_conflicting_custom_setuptools_useable(): # Here we use an older setuptools to build the pex which has a newer setuptools requirement. # These setuptools dists have different pkg_resources APIs: # $ diff \ # <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \ # <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort) # 2a3,4 # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py with temporary_dir() as resolve_cache: dists = [resolved_dist.distribution for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] interpreter = PythonInterpreter.from_binary(sys.executable, path_extras=[dist.location for dist in dists], include_site_extras=False) dists = [resolved_dist.distribution for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, 'from pkg_resources import appdirs, py31compat', dists=dists, interpreter=interpreter ) rc = PEX(pex.path()).run() assert rc == 0
def test_empty_resolve(): empty_resolve = resolve([]) assert empty_resolve == [] with temporary_dir() as td: empty_resolve = resolve([], cache=td) assert empty_resolve == []
def test_thats_it_thats_the_test(): empty_resolve = resolve([]) assert empty_resolve == set() with temporary_dir() as td: empty_resolve = resolve([], cache=td) assert empty_resolve == set()
def test_cached_dependency_pinned_unpinned_resolution_multi_run(): # This exercises the issue described here: https://github.com/pantsbuild/pex/issues/178 project1_0_0 = make_sdist(name='project', version='1.0.0') project1_1_0 = make_sdist(name='project', version='1.1.0') with temporary_dir() as td: for sdist in (project1_0_0, project1_1_0): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache dists = resolve(['project', 'project==1.0.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) assert len(dists) == 1 assert dists[0].version == '1.0.0' # This simulates separate invocations of pex but allows us to keep the same tmp cache dir Crawler.reset_cache() # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) assert len(dists) == 1 assert dists[0].version == '1.1.0' # Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact # resolvable should pull from pypi as well since inexact will and the resulting # resolvable_set.merge() would fail. Crawler.reset_cache() time.sleep(1) dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1) assert len(dists) == 1 assert dists[0].version == '1.1.0'
def test_osx_platform_intel_issue_523(): def bad_interpreter(include_site_extras=True): return PythonInterpreter.from_binary( _KNOWN_BAD_APPLE_INTERPRETER, include_site_extras=include_site_extras) interpreter = bad_interpreter(include_site_extras=False) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil. for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): for resolved_dist in resolver.resolve( [requirement], cache=cache, # We can't use wheels since we're bootstrapping them. precedence=(SourcePackage, EggPackage), interpreter=interpreter): dist = resolved_dist.distribution interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) with nested( yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, precedence=(SourcePackage, WheelPackage), interpreter=interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) args = [ '-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())' ] env = os.environ.copy() env['PEX_VERBOSE'] = '1' process = pex.run(args=args, env=env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert 0 == process.returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format( process.returncode, stderr)) # Verify this all worked under the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = '.'.join(release.split('.')[:2]) assert to_bytes( 'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def _resolve_multi(self, requirements, find_links): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() platforms = self.get_platforms(self._platforms or self._python_setup.platforms) fetchers = self._python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) context = self._python_repos.get_network_context() for platform in platforms: requirements_cache_dir = os.path.join( self._python_setup.resolver_cache_dir, str(self._interpreter.identity)) distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=self._interpreter, fetchers=fetchers, platform=platform, context=context, cache=requirements_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl) return distributions
def _resolve_multi(self, interpreter, requirements, find_links): """Multi-platform dependency resolution for PEX files. Returns a list of distributions that must be included in order to satisfy a set of requirements. That may involve distributions for multiple platforms. :param interpreter: The :class:`PythonInterpreter` to resolve for. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy the requirements on that platform. """ python_setup = PythonSetup.global_instance() python_repos = PythonRepos.global_instance() distributions = {} fetchers = python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) for platform in python_setup.platforms: requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, str(interpreter.identity)) distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, platform=None if platform == "current" else platform, context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, ) return distributions
def _resolve_multi(self, requirements, find_links): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy them on that platform. That may involve distributions for multiple platforms. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() platforms = self.get_platforms(self._platforms or self._python_setup.platforms) fetchers = self._python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) context = self._python_repos.get_network_context() for platform in platforms: requirements_cache_dir = os.path.join(self._python_setup.resolver_cache_dir, str(self._interpreter.identity)) distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=self._interpreter, fetchers=fetchers, platform=platform, context=context, cache=requirements_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl, allow_prereleases=self._python_setup.resolver_allow_prereleases) return distributions
def add_requirements(builder, cache): # type: (PEXBuilder, str) -> None for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter): builder.add_requirement(resolved_dist.requirement) builder.add_distribution(resolved_dist.distribution)
def _resolve_multi(self, requirements, find_links): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() platforms = self.get_platforms(self._platforms or self._python_setup.platforms) fetchers = self._python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) context = self._python_repos.get_network_context() for platform in platforms: distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=self._interpreter, fetchers=fetchers, platform=platform, context=context, cache=self._python_setup.resolver_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl) return distributions
def _resolve_and_link(self, interpreter, requirement, target_link): # Short-circuit if there is a local copy. if os.path.exists(target_link) and os.path.exists( os.path.realpath(target_link)): bdist = Package.from_href(os.path.realpath(target_link)) if bdist.satisfies(requirement): return bdist # Since we're resolving to bootstrap a bare interpreter, we won't have wheel available. # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into # wheels. precedence = (EggPackage, SourcePackage) distributions = resolve( requirements=[requirement], fetchers=self._python_repos.get_fetchers(), interpreter=interpreter, context=self._python_repos.get_network_context(), precedence=precedence) if not distributions: return None assert len(distributions) == 1, ( 'Expected exactly 1 distribution to be resolved for {}, ' 'found:\n\t{}'.format(requirement, '\n\t'.join(map(str, distributions)))) dist_location = distributions[0].location target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) self._logger(' installed {}'.format(target_location)) return Package.from_href(target_location)
def test_osx_platform_intel_issue_523(): def bad_interpreter(): return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored # setuptools and wheel extras. with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, precedence=(SourcePackage, WheelPackage), interpreter=pb.interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. pex = PEX(pex_file, interpreter=bad_interpreter()) def run(args, **env): pex_env = os.environ.copy() pex_env['PEX_VERBOSE'] = '1' pex_env.update(**env) process = pex.run(args=args, env=pex_env, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() return process.returncode, stdout, stderr returncode, _, stderr = run(['-c', 'import psutil']) assert 0 == returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr) ) returncode, stdout, stderr = run(['-c', 'import pkg_resources']) assert 0 != returncode, ( 'Isolated pex process succeeded but should not have found pkg-resources:\n' 'STDOUT:\n' '{}\n' 'STDERR:\n' '{}' .format(stdout, stdout, stderr) ) returncode, stdout, stderr = run( ['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'], # Let the bad interpreter site-packages setuptools leak in. PEX_INHERIT_PATH='1' ) assert 0 == returncode, ( 'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr) ) # Verify this worked along side the previously problematic pkg_resources-reported platform. release, _, _ = platform.mac_ver() major_minor = '.'.join(release.split('.')[:2]) assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
def _resolve_and_link(self, interpreter, requirement, target_link): # Short-circuit if there is a local copy. if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)): bdist = Package.from_href(os.path.realpath(target_link)) if bdist.satisfies(requirement): return bdist # Since we're resolving to bootstrap a bare interpreter, we won't have wheel available. # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into # wheels. precedence = (EggPackage, SourcePackage) distributions = resolve(requirements=[requirement], fetchers=self._python_repos.get_fetchers(), interpreter=interpreter, context=self._python_repos.get_network_context(), precedence=precedence) if not distributions: return None assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, ' 'found:\n\t{}'.format(requirement, '\n\t'.join(map(str, distributions)))) dist_location = distributions[0].location target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) self._logger(' installed {}'.format(target_location)) return Package.from_href(target_location)
def test_pex_run_conflicting_custom_setuptools_useable(): # Here we use our vendored, newer setuptools to build the pex which has an older setuptools # requirement. These setuptools dists have different pkg_resources APIs: # $ diff \ # <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \ # <(zipinfo -1 setuptools-40.6.2-py2.py3-none-any.whl | grep pkg_resources/ | sort) # 2a3,4 # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py resolved_dists = resolve(['setuptools==20.3.1']) dists = [resolved_dist.distribution for resolved_dist in resolved_dists] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, exe_contents=textwrap.dedent(""" import sys import pkg_resources try: from pkg_resources import appdirs sys.exit(1) except ImportError: pass try: from pkg_resources import py31compat sys.exit(2) except ImportError: pass """), dists=dists, ) rc = PEX(pex.path()).run(env={'PEX_VERBOSE': '9'}) assert rc == 0
def _resolve_multi(self, interpreter, requirements, find_links): """Multi-platform dependency resolution for PEX files. Returns a list of distributions that must be included in order to satisfy a set of requirements. That may involve distributions for multiple platforms. :param interpreter: The :class:`PythonInterpreter` to resolve for. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy the requirements on that platform. """ python_setup = PythonSetup.global_instance() python_repos = PythonRepos.global_instance() distributions = {} fetchers = python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) for platform in python_setup.platforms: requirements_cache_dir = os.path.join( python_setup.resolver_cache_dir, str(interpreter.identity)) distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, platform=None if platform == 'current' else platform, context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl) return distributions
def test_pex_run_custom_pex_useable(): old_pex_version = '0.7.0' resolved_dists = resolve( ['pex=={}'.format(old_pex_version), 'setuptools==40.6.3']) dists = [resolved_dist.distribution for resolved_dist in resolved_dists] with temporary_dir() as temp_dir: from pex.version import __version__ pex = write_simple_pex( temp_dir, exe_contents=textwrap.dedent(""" import sys try: # The 0.7.0 release embedded the version directly in setup.py so it should only be # available via distribution metadata. from pex.version import __version__ sys.exit(1) except ImportError: import pkg_resources dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex')) print(dist.version) """), dists=dists, ) process = PEX(pex.path()).run(blocking=False, stdout=subprocess.PIPE) stdout, _ = process.communicate() assert process.returncode == 0 assert old_pex_version == stdout.strip().decode('utf-8') assert old_pex_version != __version__
def test_simple_local_resolve(): project_sdist = make_sdist(name='project') with temporary_dir() as td: safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist))) fetchers = [Fetcher([td])] dists = resolve(['project'], fetchers=fetchers) assert len(dists) == 1
def _resolve_plugins(self): logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements))) return resolver.resolve(self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), cache=self.plugin_cache_dir, cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. allow_prereleases=PANTS_SEMVER.is_prerelease)
def _resolve_multi( self, interpreter: PythonInterpreter, requirements: List[PythonRequirement], platforms: Optional[List[Platform]], find_links: Optional[List[str]], ) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]: """Multi-platform dependency resolution for PEX files. Returns a tuple containing a list of distributions that must be included in order to satisfy a set of requirements, and the transitive == requirements for thosee distributions. This may involve distributions for multiple platforms. :param interpreter: The :class:`PythonInterpreter` to resolve for. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param platforms: A list of :class:`Platform`s to resolve for. :param find_links: Additional paths to search for source packages during resolution. :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy the requirements on that platform. """ python_setup = self._python_setup_subsystem python_repos = self._python_repos_subsystem platforms = platforms or python_setup.platforms find_links = list(find_links) if find_links else [] find_links.extend(python_repos.repos) # Individual requirements from pants may have a `repository` link attached to them, which is # extracted in `self.resolve_distributions()`. When generating a .ipex file with # `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it # tries to resolve all the requirements from BOOTSTRAP-PEX-INFO. self._all_find_links.update(OrderedSet(find_links)) distributions: Dict[str, List[Distribution]] = defaultdict(list) transitive_requirements: List[PythonRequirement] = [] all_find_links = [*python_repos.repos, *find_links] for platform in platforms: requirements_cache_dir = os.path.join( python_setup.resolver_cache_dir, str(interpreter.identity)) resolved_dists = resolve( requirements=[str(req.requirement) for req in requirements], interpreter=interpreter, platform=platform, indexes=python_repos.indexes, find_links=all_find_links, cache=requirements_cache_dir, allow_prereleases=python_setup.resolver_allow_prereleases, manylinux=python_setup.manylinux, ) for resolved_dist in resolved_dists: dist = resolved_dist.distribution transitive_requirements.append(dist.as_requirement()) distributions[platform].append(dist) return (distributions, transitive_requirements)
def test_namespace_effective(self): self.create_file('src/thrift/com/foo/one.thrift', contents=dedent(""" namespace py foo.bar struct One {} """)) one = self.make_target(spec='src/thrift/com/foo:one', target_type=PythonThriftLibrary, sources=['one.thrift']) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target( one) self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent(""" namespace py foo.baz struct Two {} """)) two = self.make_target(spec='src/thrift2/com/foo:two', target_type=PythonThriftLibrary, sources=['two.thrift']) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) python_repos = global_subsystem_instance(PythonRepos) python_setup = global_subsystem_instance(PythonSetup) interpreter_cache = PythonInterpreterCache(python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) pythonpath = [ os.path.join(get_buildroot(), t.target_base) for t in targets ] for dist in resolve( ['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], interpreter=interpreter, context=python_repos.get_network_context(), fetchers=python_repos.get_fetchers()): pythonpath.append(dist.location) process = subprocess.Popen([ interpreter.binary, '-c', 'from foo.bar.ttypes import One; from foo.baz.ttypes import Two' ], env={ 'PYTHONPATH': os.pathsep.join(pythonpath) }, stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def _resolve_plugins(self): logger.info('Resolving new plugins...:\n {}'.format('\n '.join( self._plugin_requirements))) return resolver.resolve( self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), cache=self.plugin_cache_dir, cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. allow_prereleases=PANTS_SEMVER.is_prerelease)
def assert_resolve(expected_version, **resolve_kwargs): dists = resolve( [ 'dep>=3.0.0rc1', 'dep==3.0.0rc4', ], fetchers=fetchers, **resolve_kwargs) assert 1 == len(dists) dist = dists[0] assert expected_version == dist.version
def _resolve_plugins(self): logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements))) return resolver.resolve(self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), cache=self.plugin_cache_dir, cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. allow_prereleases=PANTS_SEMVER.is_prerelease, # Plugins will all depend on `pantsbuild.pants` which is distributed as # a manylinux wheel. use_manylinux=True)
def test_pex_run_custom_setuptools_useable(): resolved_dists = resolve(['setuptools==36.2.7']) dists = [resolved_dist.distribution for resolved_dist in resolved_dists] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, 'from setuptools.sandbox import run_setup', dists=dists, ) rc = PEX(pex.path()).run() assert rc == 0
def test_pex_run_custom_setuptools_useable(): with temporary_dir() as resolve_cache: dists = resolve(['setuptools==36.2.7'], cache=resolve_cache) with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, 'from setuptools.sandbox import run_setup', dists=dists, ) rc = PEX(pex.path()).run() assert rc == 0
def test_diamond_local_resolve_cached(): # This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120 project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0']) project2_sdist = make_sdist(name='project2') with temporary_dir() as dd: for sdist in (project1_sdist, project2_sdist): safe_copy(sdist, os.path.join(dd, os.path.basename(sdist))) fetchers = [Fetcher([dd])] with temporary_dir() as cd: dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) assert len(dists) == 2
def resolve_multi(config, requirements, interpreter=None, platforms=None, conn_timeout=None, ttl=3600): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param config: Pants :class:`Config` object. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param interpreter: :class:`PythonInterpreter` for which requirements should be resolved. If None specified, defaults to current interpreter. :param platforms: Optional list of platforms against requirements will be resolved. If None specified, the defaults from `config` will be used. :param conn_timeout: Optional connection timeout for any remote fetching. :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g. "flask>=0.2" if a matching distribution is available on disk. Defaults to 3600. """ distributions = dict() interpreter = interpreter or PythonInterpreter.get() if not isinstance(interpreter, PythonInterpreter): raise TypeError( 'Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter)) install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs') platforms = get_platforms( platforms or config.getlist('python-setup', 'platforms', ['current'])) for platform in platforms: translator = Translator.default(install_cache=install_cache, interpreter=interpreter, platform=platform, conn_timeout=conn_timeout) obtainer = PantsObtainer( install_cache=install_cache, crawler=crawler_from_config(config, conn_timeout=conn_timeout), fetchers=fetchers_from_config(config) or [PyPIFetcher()], translators=translator) distributions[platform] = resolve(requirements=requirements, obtainer=obtainer, interpreter=interpreter, platform=platform) return distributions
def _resolve_plugins(self): # When bootstrapping plugins without the full pants python backend machinery in-play, we are not # guaranteed a properly initialized interpreter with wheel support so we enforce eggs only for # bdists with this custom precedence. precedence = (EggPackage, SourcePackage) logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements))) return resolver.resolve(self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), precedence=precedence, cache=self.plugin_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl)
def _resolve_plugins(self): logger.info('Resolving new plugins...:\n {}'.format('\n '.join( self._plugin_requirements))) return resolver.resolve( self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), cache=self.plugin_cache_dir, cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. allow_prereleases=PANTS_SEMVER.is_prerelease, # Plugins will all depend on `pantsbuild.pants` which is distributed as # a manylinux wheel. use_manylinux=True)
def test_namespace_effective(self): self.create_file('src/thrift/com/foo/one.thrift', contents=dedent(""" namespace py foo.bar struct One {} """)) one = self.make_target(spec='src/thrift/com/foo:one', target_type=PythonThriftLibrary, sources=['one.thrift']) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one) self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent(""" namespace py foo.baz struct Two {} """)) two = self.make_target(spec='src/thrift2/com/foo:two', target_type=PythonThriftLibrary, sources=['two.thrift']) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) python_repos = global_subsystem_instance(PythonRepos) python_setup = global_subsystem_instance(PythonSetup) interpreter_cache = PythonInterpreterCache(python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) # We need setuptools to import namespace packages (via pkg_resources), so we prime the # PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel. # TODO(John Sirois): We really should be emitting setuptools in a # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`: # https://github.com/pantsbuild/pants/issues/5975 pythonpath = interpreter.extras.values() pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets) for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], interpreter=interpreter, context=python_repos.get_network_context(), fetchers=python_repos.get_fetchers()): pythonpath.append(dist.location) process = subprocess.Popen([interpreter.binary, '-c', 'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'], env={'PYTHONPATH': os.pathsep.join(pythonpath)}, stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def resolve_multi(config, requirements, interpreter=None, platforms=None, conn_timeout=None, ttl=3600): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param config: Pants :class:`Config` object. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param interpreter: :class:`PythonInterpreter` for which requirements should be resolved. If None specified, defaults to current interpreter. :param platforms: Optional list of platforms against requirements will be resolved. If None specified, the defaults from `config` will be used. :param conn_timeout: Optional connection timeout for any remote fetching. :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g. "flask>=0.2" if a matching distribution is available on disk. Defaults to 3600. """ distributions = dict() interpreter = interpreter or PythonInterpreter.get() if not isinstance(interpreter, PythonInterpreter): raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter)) install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs') platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current'])) for platform in platforms: translator = Translator.default( install_cache=install_cache, interpreter=interpreter, platform=platform, conn_timeout=conn_timeout) obtainer = PantsObtainer( install_cache=install_cache, crawler=crawler_from_config(config, conn_timeout=conn_timeout), fetchers=fetchers_from_config(config) or [PyPIFetcher()], translators=translator) distributions[platform] = resolve(requirements=requirements, obtainer=obtainer, interpreter=interpreter, platform=platform) return distributions
def resolve_multi(config, requirements, interpreter=None, platforms=None, ttl=3600, find_links=None): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param config: Pants :class:`Config` object. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param interpreter: :class:`PythonInterpreter` for which requirements should be resolved. If None specified, defaults to current interpreter. :param platforms: Optional list of platforms against requirements will be resolved. If None specified, the defaults from `config` will be used. :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g. "flask>=0.2" if a matching distribution is available on disk. Defaults to 3600. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() interpreter = interpreter or PythonInterpreter.get() if not isinstance(interpreter, PythonInterpreter): raise TypeError( 'Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter)) cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs') platforms = get_platforms( platforms or config.getlist('python-setup', 'platforms', ['current'])) fetchers = fetchers_from_config(config) if find_links: fetchers.extend(Fetcher([path]) for path in find_links) context = context_from_config(config) for platform in platforms: distributions[platform] = resolve(requirements=requirements, interpreter=interpreter, fetchers=fetchers, platform=platform, context=context, cache=cache, cache_ttl=ttl) return distributions
def _resolve_plugins(self) -> Iterable[str]: logger.info( "Resolving new plugins...:\n {}".format("\n ".join(self._plugin_requirements)) ) resolved_dists = resolver.resolve( self._plugin_requirements, indexes=self._python_repos.indexes, find_links=self._python_repos.repos, interpreter=self._interpreter, cache=self.plugin_cache_dir, allow_prereleases=PANTS_SEMVER.is_prerelease, ) return [ self._install_plugin(resolved_dist.distribution) for resolved_dist in resolved_dists ]
def resolve_multi(python_setup, python_repos, requirements, interpreter=None, platforms=None, ttl=3600, find_links=None): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param python_setup: Pants :class:`PythonSetup` object. :param python_repos: Pants :class:`PythonRepos` object. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param interpreter: :class:`PythonInterpreter` for which requirements should be resolved. If None specified, defaults to current interpreter. :param platforms: Optional list of platforms against requirements will be resolved. If None specified, the defaults from `config` will be used. :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g. "flask>=0.2" if a matching distribution is available on disk. Defaults to 3600. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() interpreter = interpreter or PythonInterpreter.get() if not isinstance(interpreter, PythonInterpreter): raise TypeError('Expected interpreter to be a PythonInterpreter, got {}'.format(type(interpreter))) cache = os.path.join(python_setup.scratch_dir, 'eggs') platforms = get_platforms(platforms or python_setup.platforms) fetchers = python_repos.get_fetchers() if find_links: fetchers.extend(Fetcher([path]) for path in find_links) context = python_repos.get_network_context() for platform in platforms: distributions[platform] = resolve( requirements=requirements, interpreter=interpreter, fetchers=fetchers, platform=platform, context=context, cache=cache, cache_ttl=ttl) return distributions
def link_egg(repo_root, requirement): existing_dist_location = self._interpreter.get_location(requirement) if existing_dist_location is not None: existing_dist = Package.from_href(existing_dist_location) requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version) distributions = resolve([requirement], interpreter=self._interpreter, precedence=(EggPackage, SourcePackage)) self.assertEqual(1, len(distributions)) dist_location = distributions[0].location self.assertRegexpMatches(dist_location, r'\.egg$') os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location))) return Package.from_href(dist_location).raw_version
def test_ambiguous_transitive_resolvable(): # If an unbounded or larger bounded resolvable is resolved first, and a # transitive resolvable is resolved later in another round, Error(Ambiguous resolvable) can be # raised because foo pulls in foo-2.0.0 and bar->foo==1.0.0 pulls in foo-1.0.0. foo1_0 = make_sdist(name='foo', version='1.0.0') foo2_0 = make_sdist(name='foo', version='2.0.0') bar1_0 = make_sdist(name='bar', version='1.0.0', install_reqs=['foo==1.0.0']) with temporary_dir() as td: for sdist in (foo1_0, foo2_0, bar1_0): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: dists = resolve(['foo', 'bar'], fetchers=fetchers, cache=cd, cache_ttl=1000) assert len(dists) == 2 assert dists[0].version == '1.0.0'
def test_activate_extras_issue_615(): with yield_pex_builder() as pb: for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter): pb.add_requirement(resolved_dist.requirement) pb.add_dist_location(resolved_dist.distribution.location) pb.set_script('pex') pb.freeze() process = PEX(pb.path(), interpreter=pb.interpreter).run(args=['--version'], env={'PEX_VERBOSE': '9'}, blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert 0 == process.returncode, ( 'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr) ) assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()
def test_namespace_effective(self): self.create_file('src/thrift/com/foo/one.thrift', contents=dedent(""" namespace py foo.bar struct One {} """)) one = self.make_target(spec='src/thrift/com/foo:one', target_type=PythonThriftLibrary, sources=['one.thrift']) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one) self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent(""" namespace py foo.baz struct Two {} """)) two = self.make_target(spec='src/thrift2/com/foo:two', target_type=PythonThriftLibrary, sources=['two.thrift']) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) python_repos = global_subsystem_instance(PythonRepos) python_setup = global_subsystem_instance(PythonSetup) interpreter_cache = PythonInterpreterCache(python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets] for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], interpreter=interpreter, context=python_repos.get_network_context(), fetchers=python_repos.get_fetchers()): pythonpath.append(dist.location) process = subprocess.Popen([interpreter.binary, '-c', 'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'], env={'PYTHONPATH': os.pathsep.join(pythonpath)}, stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def _resolve_multi(self, interpreter, requirements, platforms, find_links): """Multi-platform dependency resolution for PEX files. Returns a list of distributions that must be included in order to satisfy a set of requirements. That may involve distributions for multiple platforms. :param interpreter: The :class:`PythonInterpreter` to resolve for. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param platforms: A list of :class:`Platform`s to resolve for. :param find_links: Additional paths to search for source packages during resolution. :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy the requirements on that platform. """ python_setup = self._python_setup_subsystem python_repos = self._python_repos_subsystem platforms = platforms or python_setup.platforms find_links = find_links or [] distributions = {} fetchers = python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) for platform in platforms: requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, str(interpreter.identity)) resolved_dists = resolve( requirements=[str(req.requirement) for req in requirements], interpreter=interpreter, fetchers=fetchers, platform=platform, context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, allow_prereleases=python_setup.resolver_allow_prereleases, use_manylinux=python_setup.use_manylinux) distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists] return distributions