def _resolve_and_link(config, requirement, target_link, installer_provider, logger=print): # Short-circuit if there is a local copy if os.path.exists(target_link) and os.path.exists( os.path.realpath(target_link)): egg = EggPackage(os.path.realpath(target_link)) if egg.satisfies(requirement): return egg fetchers = fetchers_from_config(config) context = context_from_config(config) iterator = Iterator(fetchers=fetchers, crawler=Crawler(context)) links = [ link for link in iterator.iter(requirement) if isinstance(link, SourcePackage) ] for link in links: logger(' fetching %s' % link.url) sdist = context.fetch(link) logger(' installing %s' % sdist) installer = installer_provider(sdist) dist_location = installer.bdist() target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) logger(' installed %s' % target_location) return EggPackage(target_location)
def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider): # Short-circuit if there is a local copy if os.path.exists(target_link) and os.path.exists( os.path.realpath(target_link)): egg = EggPackage(os.path.realpath(target_link)) if egg.satisfies(requirement): return egg context = Context.get() iterator = Iterator(fetchers=fetchers, crawler=Crawler(context)) links = [ link for link in iterator.iter(requirement) if isinstance(link, SourcePackage) ] with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2): for link in links: with TRACER.timed('Fetching %s' % link, V=3): sdist = context.fetch(link) with TRACER.timed('Installing %s' % link, V=3): installer = installer_provider(sdist) dist_location = installer.bdist() target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) return EggPackage(target_location)
def get_iterator(self): return Iterator( fetchers=self._fetchers, crawler=self.get_crawler(), follow_links=self._allow_external, allow_prereleases=self._allow_prereleases )
def test_empty_iteration(): crawler_mock = mock.create_autospec(Crawler, spec_set=True) crawler_mock.crawl.return_value = [] iterator = Iterator(crawler=crawler_mock) assert list(iterator.iter(Requirement.parse('foo'))) == [] assert len(crawler_mock.crawl.mock_calls) == 1 _, args, kwargs = crawler_mock.crawl.mock_calls[0] assert list(args[0]) == list(PyPIFetcher().urls(Requirement.parse('foo'))) assert kwargs == {'follow_links': False}
def test_iteration_with_return(): pex_url = 'https://pypi.python.org/packages/source/p/pex/pex-0.8.6.tar.gz' crawler_mock = mock.create_autospec(Crawler, spec_set=True) crawler_mock.crawl.return_value = [pex_url] iterator = Iterator(crawler=crawler_mock, follow_links=True) assert list(iterator.iter( Requirement.parse('pex'))) == [SourcePackage(pex_url)] assert len(crawler_mock.crawl.mock_calls) == 1 _, _, kwargs = crawler_mock.crawl.mock_calls[0] assert kwargs == {'follow_links': True}
def package_iterator(self, resolvable, existing=None): iterator = Iterator(fetchers=[Fetcher([self.__cache])], allow_prereleases=self._allow_prereleases) packages = self.filter_packages_by_supported_tags(resolvable.compatible(iterator)) if packages and self.__cache_ttl: packages = self.filter_packages_by_ttl(packages, self.__cache_ttl) return itertools.chain( packages, super(CachingResolver, self).package_iterator(resolvable, existing=existing) )
def assert_iteration(all_versions, *expected_versions, **iterator_kwargs): def package_url(version): return 'https://pypi.org/packages/source/p/pex/pex-%s.tar.gz' % version urls = [package_url(v) for v in all_versions] crawler_mock = mock.create_autospec(Crawler, spec_set=True) crawler_mock.crawl.return_value = urls iterator = Iterator(crawler=crawler_mock, follow_links=True, **iterator_kwargs) assert list(iterator.iter(Requirement.parse('pex'))) == [ SourcePackage(package_url(v)) for v in expected_versions ] assert len(crawler_mock.crawl.mock_calls) == 1 _, _, kwargs = crawler_mock.crawl.mock_calls[0] assert kwargs == {'follow_links': True}
def test_href_translation(): VERSIONS = ['0.4.0', '0.4.1', '0.5.0', '0.6.0'] def fake_link(version): return 'http://www.example.com/foo/bar/psutil-%s.tar.gz' % version fc = FakeCrawler([fake_link(v) for v in VERSIONS]) ob = Iterator(crawler=fc) for v in VERSIONS: pkgs = list(ob.iter(Requirement.parse('psutil==%s' % v))) assert len(pkgs) == 1, 'Version: %s' % v assert pkgs[0] == SourcePackage(fake_link(v)) assert list(ob.iter(Requirement.parse('psutil>=0.5.0'))) == [ SourcePackage(fake_link('0.6.0')), SourcePackage(fake_link('0.5.0')) ] assert list(ob.iter(Requirement.parse('psutil'))) == [ SourcePackage(fake_link(v)) for v in reversed(VERSIONS) ]
def _resolve_and_link(self, requirement, target_link, installer_provider): # Short-circuit if there is a local copy. if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)): egg = EggPackage(os.path.realpath(target_link)) if egg.satisfies(requirement): return egg fetchers = self._python_repos.get_fetchers() context = self._python_repos.get_network_context() iterator = Iterator(fetchers=fetchers, crawler=Crawler(context)) links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)] for link in links: self._logger(' fetching {}'.format(link.url)) sdist = context.fetch(link) self._logger(' installing {}'.format(sdist)) installer = installer_provider(sdist) dist_location = installer.bdist() target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) self._logger(' installed {}'.format(target_location)) return EggPackage(target_location)
def test_package_precedence(): source = SourcePackage('psutil-0.6.1.tar.gz') egg = EggPackage('psutil-0.6.1-py2.6.egg') whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl') # default precedence assert Iterator.package_precedence(whl) > Iterator.package_precedence(egg) assert Iterator.package_precedence(egg) > Iterator.package_precedence( source) assert Iterator.package_precedence(whl) > Iterator.package_precedence( source) # overridden precedence PRECEDENCE = (EggPackage, WheelPackage) assert Iterator.package_precedence(source, PRECEDENCE) == (source.version, -1, True ) # unknown rank assert Iterator.package_precedence( whl, PRECEDENCE) > Iterator.package_precedence(source, PRECEDENCE) assert Iterator.package_precedence( egg, PRECEDENCE) > Iterator.package_precedence(whl, PRECEDENCE)
def test_package_precedence(): source = SourcePackage('psutil-0.6.1.tar.gz') egg = EggPackage('psutil-0.6.1-py2.6.egg') whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl') # default precedence assert Iterator.package_precedence(whl) > Iterator.package_precedence(egg) assert Iterator.package_precedence(egg) > Iterator.package_precedence(source) assert Iterator.package_precedence(whl) > Iterator.package_precedence(source) # overridden precedence PRECEDENCE = (EggPackage, WheelPackage) assert Iterator.package_precedence(source, PRECEDENCE) == ( source.version, -1, True) # unknown rank assert Iterator.package_precedence(whl, PRECEDENCE) > Iterator.package_precedence( source, PRECEDENCE) assert Iterator.package_precedence(egg, PRECEDENCE) > Iterator.package_precedence( whl, PRECEDENCE)