def test_finder_priority_page_over_deplink():
"""Test PackageFinder prefers page links over equivalent dependency links"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder([], ["http://pypi.python.org/simple"])
finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("http://pypi")
def test_finder_priority_page_over_deplink():
"""Test PackageFinder prefers page links over equivalent dependency links"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder([], ["https://pypi.python.org/simple"])
finder.add_dependency_links(['https://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_deplink():
"""
Test PackageFinder with dependency links only
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
[],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(
['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_deplink():
"""
Test PackageFinder with dependency links only
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
[],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(
['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_priority_page_over_deplink():
"""
Test PackageFinder prefers page links over equivalent dependency links
"""
req = InstallRequirement.from_line('pip==1.5.6', None)
finder = PackageFinder(
[],
["https://pypi.python.org/simple"],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links([
'https://warehouse.python.org/packages/source/p/pip/pip-1.5.6.tar.gz'])
all_versions = finder._find_all_versions(req.name)
# Check that the dependency_link is last
assert all_versions[-1].location.url.startswith('https://warehouse')
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_priority_page_over_deplink():
"""
Test PackageFinder prefers page links over equivalent dependency links
"""
req = InstallRequirement.from_line('pip==1.5.6', None)
finder = PackageFinder(
[],
["https://pypi.python.org/simple"],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links([
'https://warehouse.python.org/packages/source/p/pip/pip-1.5.6.tar.gz'])
all_versions = finder._find_all_versions(req.name)
# Check that the dependency_link is last
assert all_versions[-1].location.url.startswith('https://warehouse')
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
class PackageManager(object):
"""Interface to packages."""
def __init__(self,
overrides={},
versions=[],
extra=(),
dependency_links=[],
exe=sys.executable,
python_path="",
download_cache_root="",
cache=None,
link_hook=lambda overrides, spec, link: (link, None),
dependency_hook=lambda overrides, spec, deps, package: deps,
spec_hook=lambda overrides, spec: spec):
self.extra = extra
self.exe, self.python_path = exe, python_path
self.overrides = overrides or {}
self.versions = versions or []
self._dependency_hook = dependency_hook
self._link_hook = link_hook
self._spec_hook = spec_hook
self.finder = PackageFinder(
find_links=[],
index_urls=['https://pypi.python.org/simple/'],
use_mirrors=True,
mirrors=[],
allow_all_external=True,
allow_all_insecure=True)
self.finder.add_dependency_links(dependency_links)
self.download_cache_root = download_cache_root
cache = cache or defaultdict(dict)
self._link_cache = cache["link_cache"]
self._dep_cache = cache["dep_cache"]
self._pkg_info_cache = cache["pkg_info_cache"]
self._extract_cache = cache["extract_cache"]
self._best_match_call_cache = {}
self._dep_call_cache = {}
self._pkg_info_call_cache = {}
def find_best_match(self, spec):
def _find_cached_match(spec):
#if spec.is_pinned:
## If this is a pinned spec, we can take a shortcut: if it is
## found in the dependency cache, we can safely assume it has
## been downloaded before, and thus must exist. We can know
## this without every reaching out to PyPI and avoid the
## network overhead.
#name, version = spec.name, first(spec.preds)[1]
#if (name, version) in self._dep_cache:
#source = 'dependency cache'
#return version, source
version = None
overrides = self.overrides.get(spec.name)
## Try the link cache, and otherwise, try PyPI
if (spec.no_extra, overrides) in self._link_cache:
link, version = self._link_cache[(spec.no_extra, overrides)]
source = 'link cache'
else:
try:
requirement = InstallRequirement.from_line(specline)
link = self.finder.find_requirement(requirement, False)
except DistributionNotFound:
requirement = InstallRequirement.from_line(
specline, prereleases=True)
link = self.finder.find_requirement(requirement, False)
link, version = self._link_hook(overrides, spec, link)
# Hack to make pickle work
link.comes_from = None
source = 'PyPI'
if link.egg_fragment:
version = link.egg_fragment.rsplit('-', 1)[1]
link = Link(link.url_without_fragment +
"#%s=%s" % self.get_hash(link))
elif not version:
_, version = splitext(link.filename)[0].rsplit('-', 1)
# It's more reliable to get version from pinned spec then filename
if spec.is_pinned:
version = spec.pinned
assert version, "Version must be set!"
self._link_cache[(spec.no_extra, overrides)] = (link, version)
# Take this moment to smartly insert the pinned variant of this
# spec into the link_cache, too
pinned_spec = Spec.from_pinned(spec.name, version)
self._link_cache[pinned_spec.fullname] = (link, version)
return version, source
version = next((v for v in self.versions if v.name == spec.name), None)
if version:
spec.pinned = version.pinned
specline = spec.no_extra
if '==' not in specline or specline not in self._best_match_call_cache:
logger.debug('- Finding best package matching %s' % spec)
version = next((v for v in self.versions if v.name == spec.name), None)
with logger.indent():
version, source = _find_cached_match(spec)
if '==' not in specline or specline not in self._best_match_call_cache:
logger.debug(' Found best match: %s (from %s)' %
(version, source))
self._best_match_call_cache[specline] = True
return version
def get_dependencies(self, name, version, extra=()):
"""Gets list of dependencies from package"""
spec = Spec.from_pinned(name, version, extra=extra)
overrides = self.overrides.get(spec.name)
extra = self.extra + extra
if spec not in self._dep_call_cache:
logger.debug('- Getting dependencies for %s-%s' % (name, version))
with logger.indent():
deps = self._dep_cache.get((spec, overrides))
links = self._dep_cache.get((spec, overrides, "links"))
if deps is not None and links is not None:
source = 'dependency cache'
else:
package = self.get_package(spec)
deps = package.get_deps(extra=extra)
deps = self._dependency_hook(overrides, spec, deps, package)
self._dep_cache[(spec, overrides)] = deps
links = package.get_dependency_links()
self._dep_cache[(spec, overrides, "links")] = links
source = 'package archive'
# Run spec hook
deps = [(self._spec_hook(self.overrides.get(dep.name), dep),
src) if self.overrides.get(dep.name) else (dep, src)
for dep, src in deps]
if spec not in self._dep_call_cache:
logger.debug(' Found: %s (from %s)' % (deps, source))
# At this point do not forget to add dependency links
self.finder.add_dependency_links(links)
self._dep_call_cache[spec] = True
return deps
def get_pkg_info(self, name, version):
spec = Spec.from_pinned(name, version)
if spec.no_extra not in self._pkg_info_call_cache:
logger.debug('- Getting pkginfo for %s-%s' % (name, version))
with logger.indent():
pkg_info = self._pkg_info_cache.get(spec.no_extra)
if pkg_info is not None:
source = 'pkg_info cache'
else:
package = self.get_package(spec)
pkg_info = package.get_pkginfo()
pkg_info["has_tests"] = package.has_tests()
self._pkg_info_cache[spec.no_extra] = pkg_info
source = 'package archive'
if spec.no_extra not in self._pkg_info_call_cache:
logger.debug(' Found pkg_info (from %s)' % (source, ))
self._pkg_info_call_cache[spec.no_extra] = True
return pkg_info
def get_link(self, name, version):
logger.debug('- Getting link for %s-%s' % (name, version))
spec = Spec.from_pinned(name, version)
self.find_best_match(spec)
return self._link_cache[spec.fullname]
def get_hash(self, link):
if link.hash and link.hash_name:
return (link.hash_name, link.hash)
def md5hash(path):
return ("md5", hashlib.md5(open(path, 'rb').read()).hexdigest())
url = link.url_without_fragment
logger.info('- Hashing package on url %s' % (url, ))
with logger.indent():
fullpath = self._get_local_package_path(link.url_without_fragment)
if os.path.exists(fullpath):
logger.info(' Archive cache hit: {0}'.format(link.filename))
return md5hash(fullpath)
return md5hash(self._download_package(link))
def get_package(self, spec):
path = self._get_or_download_package(spec.fullname)
return Package(package_dir=self._extract(path),
exe=self.exe,
python_path=self.python_path)
# Helper methods
def _get_local_package_path(self, url): # noqa
"""Returns the full local path name for a given URL. This
does not require the package archive to exist locally. In fact, this
can be used to calculate the destination path for a download.
"""
cache_key = quote(url, '')
fullpath = os.path.join(self.download_cache_root, cache_key)
return fullpath
def _get_or_download_package(self, specline):
"""Returns the local path from the package cache, downloading as
needed.
"""
logger.debug('- Getting package location for %s' % (specline, ))
with logger.indent():
link, version = self._link_cache[specline]
fullpath = self._get_local_package_path(link.url_without_fragment)
if os.path.exists(fullpath):
logger.info(' Archive cache hit: {0}'.format(link.filename))
return fullpath
logger.info(' Archive cache miss, downloading {0}...'.format(
link.filename))
return self._download_package(link)
def _download_package(self, link):
"""Downloads the given package link contents to the local
package cache. Overwrites anything that's in the cache already.
"""
url = link.url_without_fragment
logger.info('- Downloading package from %s' % (url, ))
with logger.indent():
fullpath = self._get_local_package_path(url)
response = _get_response_from_url(url, link)
_download_url(response, link, fullpath)
return fullpath
def _unpack_archive(self, path, target_directory):
logger.debug('- Unpacking %s' % (path, ))
with logger.indent():
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
else:
archive = tarfile.open(path)
try:
archive.extractall(target_directory)
except IOError:
logger.error("Error extracting %s" % (path, ))
raise
finally:
archive.close()
def _extract(self, path):
if path in self._extract_cache:
return self._extract_cache[path]
logger.info('- Extracting package %s' % (path, ))
build_dir = tempfile.mkdtemp()
atexit.register(shutil.rmtree, build_dir)
unpack_dir = os.path.join(build_dir, 'build')
self._unpack_archive(path, unpack_dir)
# Cache unpack
self._extract_cache[path] = unpack_dir
return unpack_dir
class PackageManager(object):
"""Interface to packages."""
def __init__(
self, overrides={}, versions=[], extra=(), dependency_links=[],
exe=sys.executable, python_path="",
download_cache_root="", cache=None,
link_hook=lambda overrides, spec, link: (link, None),
dependency_hook=lambda overrides, spec, deps, package: deps,
spec_hook=lambda overrides, spec: spec
):
self.extra = extra
self.exe, self.python_path = exe, python_path
self.overrides = overrides or {}
self.versions = versions or []
self._dependency_hook = dependency_hook
self._link_hook = link_hook
self._spec_hook = spec_hook
self.finder = PackageFinder(
find_links=[],
index_urls=['https://pypi.python.org/simple/'],
use_mirrors=True,
mirrors=[],
allow_all_external=True,
allow_all_insecure=True
)
self.finder.add_dependency_links(dependency_links)
self.download_cache_root = download_cache_root
cache = cache or defaultdict(dict)
self._link_cache = cache["link_cache"]
self._dep_cache = cache["dep_cache"]
self._pkg_info_cache = cache["pkg_info_cache"]
self._extract_cache = cache["extract_cache"]
self._best_match_call_cache = {}
self._dep_call_cache = {}
self._pkg_info_call_cache = {}
def find_best_match(self, spec):
def _find_cached_match(spec):
#if spec.is_pinned:
## If this is a pinned spec, we can take a shortcut: if it is
## found in the dependency cache, we can safely assume it has
## been downloaded before, and thus must exist. We can know
## this without every reaching out to PyPI and avoid the
## network overhead.
#name, version = spec.name, first(spec.preds)[1]
#if (name, version) in self._dep_cache:
#source = 'dependency cache'
#return version, source
version = None
overrides = self.overrides.get(spec.name)
## Try the link cache, and otherwise, try PyPI
if (spec.no_extra, overrides) in self._link_cache:
link, version = self._link_cache[(spec.no_extra, overrides)]
source = 'link cache'
else:
try:
requirement = InstallRequirement.from_line(specline)
link = self.finder.find_requirement(requirement, False)
except DistributionNotFound:
requirement = InstallRequirement.from_line(
specline, prereleases=True)
link = self.finder.find_requirement(requirement, False)
link, version = self._link_hook(overrides, spec, link)
# Hack to make pickle work
link.comes_from = None
source = 'PyPI'
if link.egg_fragment:
version = link.egg_fragment.rsplit('-', 1)[1]
link = Link(
link.url_without_fragment + "#%s=%s" % self.get_hash(link)
)
elif not version:
_, version = splitext(link.filename)[0].rsplit('-', 1)
# It's more reliable to get version from pinned spec then filename
if spec.is_pinned:
version = spec.pinned
assert version, "Version must be set!"
self._link_cache[(spec.no_extra, overrides)] = (link, version)
# Take this moment to smartly insert the pinned variant of this
# spec into the link_cache, too
pinned_spec = Spec.from_pinned(spec.name, version)
self._link_cache[pinned_spec.fullname] = (link, version)
return version, source
version = next((v for v in self.versions if v.name == spec.name), None)
if version:
spec.pinned = version.pinned
specline = spec.no_extra
if '==' not in specline or specline not in self._best_match_call_cache:
logger.debug('- Finding best package matching %s' % spec)
version = next((v for v in self.versions if v.name == spec.name), None)
with logger.indent():
version, source = _find_cached_match(spec)
if '==' not in specline or specline not in self._best_match_call_cache:
logger.debug(' Found best match: %s (from %s)' % (version, source))
self._best_match_call_cache[specline] = True
return version
def get_dependencies(self, name, version, extra=()):
"""Gets list of dependencies from package"""
spec = Spec.from_pinned(name, version, extra=extra)
overrides = self.overrides.get(spec.name)
extra = self.extra + extra
if spec not in self._dep_call_cache:
logger.debug('- Getting dependencies for %s-%s' % (name, version))
with logger.indent():
deps = self._dep_cache.get((spec, overrides))
links = self._dep_cache.get((spec, overrides, "links"))
if deps is not None and links is not None:
source = 'dependency cache'
else:
package = self.get_package(spec)
deps = package.get_deps(extra=extra)
deps = self._dependency_hook(overrides, spec, deps, package)
self._dep_cache[(spec, overrides)] = deps
links = package.get_dependency_links()
self._dep_cache[(spec, overrides, "links")] = links
source = 'package archive'
# Run spec hook
deps = [
(self._spec_hook(self.overrides.get(dep.name), dep), src)
if self.overrides.get(dep.name) else (dep, src)
for dep, src in deps
]
if spec not in self._dep_call_cache:
logger.debug(' Found: %s (from %s)' % (deps, source))
# At this point do not forget to add dependency links
self.finder.add_dependency_links(links)
self._dep_call_cache[spec] = True
return deps
def get_pkg_info(self, name, version):
spec = Spec.from_pinned(name, version)
if spec.no_extra not in self._pkg_info_call_cache:
logger.debug('- Getting pkginfo for %s-%s' % (name, version))
with logger.indent():
pkg_info = self._pkg_info_cache.get(spec.no_extra)
if pkg_info is not None:
source = 'pkg_info cache'
else:
package = self.get_package(spec)
pkg_info = package.get_pkginfo()
pkg_info["has_tests"] = package.has_tests()
self._pkg_info_cache[spec.no_extra] = pkg_info
source = 'package archive'
if spec.no_extra not in self._pkg_info_call_cache:
logger.debug(' Found pkg_info (from %s)' % (source,))
self._pkg_info_call_cache[spec.no_extra] = True
return pkg_info
def get_link(self, name, version):
logger.debug('- Getting link for %s-%s' % (name, version))
spec = Spec.from_pinned(name, version)
self.find_best_match(spec)
return self._link_cache[spec.fullname]
def get_hash(self, link):
if link.hash and link.hash_name:
return (link.hash_name, link.hash)
def md5hash(path):
return ("md5", hashlib.md5(open(path, 'rb').read()).hexdigest())
url = link.url_without_fragment
logger.info('- Hashing package on url %s' % (url,))
with logger.indent():
fullpath = self._get_local_package_path(link.url_without_fragment)
if os.path.exists(fullpath):
logger.info(' Archive cache hit: {0}'.format(link.filename))
return md5hash(fullpath)
return md5hash(self._download_package(link))
def get_package(self, spec):
path = self._get_or_download_package(spec.fullname)
return Package(
package_dir=self._extract(path),
exe=self.exe, python_path=self.python_path
)
# Helper methods
def _get_local_package_path(self, url): # noqa
"""Returns the full local path name for a given URL. This
does not require the package archive to exist locally. In fact, this
can be used to calculate the destination path for a download.
"""
cache_key = quote(url, '')
fullpath = os.path.join(self.download_cache_root, cache_key)
return fullpath
def _get_or_download_package(self, specline):
"""Returns the local path from the package cache, downloading as
needed.
"""
logger.debug('- Getting package location for %s' % (specline,))
with logger.indent():
link, version = self._link_cache[specline]
fullpath = self._get_local_package_path(link.url_without_fragment)
if os.path.exists(fullpath):
logger.info(' Archive cache hit: {0}'.format(link.filename))
return fullpath
logger.info(' Archive cache miss, downloading {0}...'.format(
link.filename
))
return self._download_package(link)
def _download_package(self, link):
"""Downloads the given package link contents to the local
package cache. Overwrites anything that's in the cache already.
"""
url = link.url_without_fragment
logger.info('- Downloading package from %s' % (url,))
with logger.indent():
fullpath = self._get_local_package_path(url)
response = _get_response_from_url(url, link)
_download_url(response, link, fullpath)
return fullpath
def _unpack_archive(self, path, target_directory):
logger.debug('- Unpacking %s' % (path,))
with logger.indent():
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
else:
archive = tarfile.open(path)
try:
archive.extractall(target_directory)
except IOError:
logger.error("Error extracting %s" % (path,))
raise
finally:
archive.close()
def _extract(self, path):
if path in self._extract_cache:
return self._extract_cache[path]
logger.info('- Extracting package %s' % (path,))
build_dir = tempfile.mkdtemp()
atexit.register(shutil.rmtree, build_dir)
unpack_dir = os.path.join(build_dir, 'build')
self._unpack_archive(path, unpack_dir)
# Cache unpack
self._extract_cache[path] = unpack_dir
return unpack_dir
