def _resolve_and_link(config,
requirement,
target_link,
installer_provider,
logger=print):
# Short-circuit if there is a local copy
if os.path.exists(target_link) and os.path.exists(
os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
fetchers = fetchers_from_config(config)
context = context_from_config(config)
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [
link for link in iterator.iter(requirement)
if isinstance(link, SourcePackage)
]
for link in links:
logger(' fetching %s' % link.url)
sdist = context.fetch(link)
logger(' installing %s' % sdist)
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(os.path.dirname(target_link),
os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
logger(' installed %s' % target_location)
return EggPackage(target_location)
def _resolve_and_link_interpreter(requirement, fetchers, target_link,
installer_provider):
# Short-circuit if there is a local copy
if os.path.exists(target_link) and os.path.exists(
os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
context = Context.get()
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [
link for link in iterator.iter(requirement)
if isinstance(link, SourcePackage)
]
with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2):
for link in links:
with TRACER.timed('Fetching %s' % link, V=3):
sdist = context.fetch(link)
with TRACER.timed('Installing %s' % link, V=3):
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(os.path.dirname(target_link),
os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
return EggPackage(target_location)
def get_iterator(self):
return Iterator(
fetchers=self._fetchers,
crawler=self.get_crawler(),
follow_links=self._allow_external,
allow_prereleases=self._allow_prereleases
)
def test_empty_iteration():
crawler_mock = mock.create_autospec(Crawler, spec_set=True)
crawler_mock.crawl.return_value = []
iterator = Iterator(crawler=crawler_mock)
assert list(iterator.iter(Requirement.parse('foo'))) == []
assert len(crawler_mock.crawl.mock_calls) == 1
_, args, kwargs = crawler_mock.crawl.mock_calls[0]
assert list(args[0]) == list(PyPIFetcher().urls(Requirement.parse('foo')))
assert kwargs == {'follow_links': False}
def test_iteration_with_return():
pex_url = 'https://pypi.python.org/packages/source/p/pex/pex-0.8.6.tar.gz'
crawler_mock = mock.create_autospec(Crawler, spec_set=True)
crawler_mock.crawl.return_value = [pex_url]
iterator = Iterator(crawler=crawler_mock, follow_links=True)
assert list(iterator.iter(
Requirement.parse('pex'))) == [SourcePackage(pex_url)]
assert len(crawler_mock.crawl.mock_calls) == 1
_, _, kwargs = crawler_mock.crawl.mock_calls[0]
assert kwargs == {'follow_links': True}
def package_iterator(self, resolvable, existing=None):
iterator = Iterator(fetchers=[Fetcher([self.__cache])],
allow_prereleases=self._allow_prereleases)
packages = self.filter_packages_by_supported_tags(resolvable.compatible(iterator))
if packages and self.__cache_ttl:
packages = self.filter_packages_by_ttl(packages, self.__cache_ttl)
return itertools.chain(
packages,
super(CachingResolver, self).package_iterator(resolvable, existing=existing)
)
def assert_iteration(all_versions, *expected_versions, **iterator_kwargs):
def package_url(version):
return 'https://pypi.org/packages/source/p/pex/pex-%s.tar.gz' % version
urls = [package_url(v) for v in all_versions]
crawler_mock = mock.create_autospec(Crawler, spec_set=True)
crawler_mock.crawl.return_value = urls
iterator = Iterator(crawler=crawler_mock,
follow_links=True,
**iterator_kwargs)
assert list(iterator.iter(Requirement.parse('pex'))) == [
SourcePackage(package_url(v)) for v in expected_versions
]
assert len(crawler_mock.crawl.mock_calls) == 1
_, _, kwargs = crawler_mock.crawl.mock_calls[0]
assert kwargs == {'follow_links': True}
def test_href_translation():
VERSIONS = ['0.4.0', '0.4.1', '0.5.0', '0.6.0']
def fake_link(version):
return 'http://www.example.com/foo/bar/psutil-%s.tar.gz' % version
fc = FakeCrawler([fake_link(v) for v in VERSIONS])
ob = Iterator(crawler=fc)
for v in VERSIONS:
pkgs = list(ob.iter(Requirement.parse('psutil==%s' % v)))
assert len(pkgs) == 1, 'Version: %s' % v
assert pkgs[0] == SourcePackage(fake_link(v))
assert list(ob.iter(Requirement.parse('psutil>=0.5.0'))) == [
SourcePackage(fake_link('0.6.0')),
SourcePackage(fake_link('0.5.0'))
]
assert list(ob.iter(Requirement.parse('psutil'))) == [
SourcePackage(fake_link(v)) for v in reversed(VERSIONS)
]
def _resolve_and_link(self, requirement, target_link, installer_provider):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
fetchers = self._python_repos.get_fetchers()
context = self._python_repos.get_network_context()
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]
for link in links:
self._logger(' fetching {}'.format(link.url))
sdist = context.fetch(link)
self._logger(' installing {}'.format(sdist))
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
self._logger(' installed {}'.format(target_location))
return EggPackage(target_location)
def test_package_precedence():
source = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py2.6.egg')
whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl')
# default precedence
assert Iterator.package_precedence(whl) > Iterator.package_precedence(egg)
assert Iterator.package_precedence(egg) > Iterator.package_precedence(
source)
assert Iterator.package_precedence(whl) > Iterator.package_precedence(
source)
# overridden precedence
PRECEDENCE = (EggPackage, WheelPackage)
assert Iterator.package_precedence(source,
PRECEDENCE) == (source.version, -1, True
) # unknown rank
assert Iterator.package_precedence(
whl, PRECEDENCE) > Iterator.package_precedence(source, PRECEDENCE)
assert Iterator.package_precedence(
egg, PRECEDENCE) > Iterator.package_precedence(whl, PRECEDENCE)
def test_package_precedence():
source = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py2.6.egg')
whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl')
# default precedence
assert Iterator.package_precedence(whl) > Iterator.package_precedence(egg)
assert Iterator.package_precedence(egg) > Iterator.package_precedence(source)
assert Iterator.package_precedence(whl) > Iterator.package_precedence(source)
# overridden precedence
PRECEDENCE = (EggPackage, WheelPackage)
assert Iterator.package_precedence(source, PRECEDENCE) == (
source.version, -1, True) # unknown rank
assert Iterator.package_precedence(whl, PRECEDENCE) > Iterator.package_precedence(
source, PRECEDENCE)
assert Iterator.package_precedence(egg, PRECEDENCE) > Iterator.package_precedence(
whl, PRECEDENCE)
