def test_finder_priority_page_over_deplink(): """Test PackageFinder prefers page links over equivalent dependency links""" req = InstallRequirement.from_line('gmpy==1.15', None) finder = PackageFinder([], ["http://pypi.python.org/simple"]) finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/']) link = finder.find_requirement(req, False) assert link.url.startswith("http://pypi")
def test_finder_priority_page_over_deplink(): """Test PackageFinder prefers page links over equivalent dependency links""" req = InstallRequirement.from_line('gmpy==1.15', None) finder = PackageFinder([], ["https://pypi.python.org/simple"]) finder.add_dependency_links(['https://c.pypi.python.org/simple/gmpy/']) link = finder.find_requirement(req, False) assert link.url.startswith("https://pypi"), link
def test_finder_deplink(): """ Test PackageFinder with dependency links only """ req = InstallRequirement.from_line('gmpy==1.15', None) finder = PackageFinder( [], [], process_dependency_links=True, session=PipSession(), ) finder.add_dependency_links( ['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip']) link = finder.find_requirement(req, False) assert link.url.startswith("https://pypi"), link
def test_finder_deplink(): """ Test PackageFinder with dependency links only """ req = InstallRequirement.from_line('gmpy==1.15', None) finder = PackageFinder( [], [], process_dependency_links=True, session=PipSession(), ) finder.add_dependency_links( ['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip']) link = finder.find_requirement(req, False) assert link.url.startswith("https://pypi"), link
def test_finder_priority_page_over_deplink(): """ Test PackageFinder prefers page links over equivalent dependency links """ req = InstallRequirement.from_line('pip==1.5.6', None) finder = PackageFinder( [], ["https://pypi.python.org/simple"], process_dependency_links=True, session=PipSession(), ) finder.add_dependency_links([ 'https://warehouse.python.org/packages/source/p/pip/pip-1.5.6.tar.gz']) all_versions = finder._find_all_versions(req.name) # Check that the dependency_link is last assert all_versions[-1].location.url.startswith('https://warehouse') link = finder.find_requirement(req, False) assert link.url.startswith("https://pypi"), link
def test_finder_priority_page_over_deplink(): """ Test PackageFinder prefers page links over equivalent dependency links """ req = InstallRequirement.from_line('pip==1.5.6', None) finder = PackageFinder( [], ["https://pypi.python.org/simple"], process_dependency_links=True, session=PipSession(), ) finder.add_dependency_links([ 'https://warehouse.python.org/packages/source/p/pip/pip-1.5.6.tar.gz']) all_versions = finder._find_all_versions(req.name) # Check that the dependency_link is last assert all_versions[-1].location.url.startswith('https://warehouse') link = finder.find_requirement(req, False) assert link.url.startswith("https://pypi"), link
class PackageManager(object): """Interface to packages.""" def __init__(self, overrides={}, versions=[], extra=(), dependency_links=[], exe=sys.executable, python_path="", download_cache_root="", cache=None, link_hook=lambda overrides, spec, link: (link, None), dependency_hook=lambda overrides, spec, deps, package: deps, spec_hook=lambda overrides, spec: spec): self.extra = extra self.exe, self.python_path = exe, python_path self.overrides = overrides or {} self.versions = versions or [] self._dependency_hook = dependency_hook self._link_hook = link_hook self._spec_hook = spec_hook self.finder = PackageFinder( find_links=[], index_urls=['https://pypi.python.org/simple/'], use_mirrors=True, mirrors=[], allow_all_external=True, allow_all_insecure=True) self.finder.add_dependency_links(dependency_links) self.download_cache_root = download_cache_root cache = cache or defaultdict(dict) self._link_cache = cache["link_cache"] self._dep_cache = cache["dep_cache"] self._pkg_info_cache = cache["pkg_info_cache"] self._extract_cache = cache["extract_cache"] self._best_match_call_cache = {} self._dep_call_cache = {} self._pkg_info_call_cache = {} def find_best_match(self, spec): def _find_cached_match(spec): #if spec.is_pinned: ## If this is a pinned spec, we can take a shortcut: if it is ## found in the dependency cache, we can safely assume it has ## been downloaded before, and thus must exist. We can know ## this without every reaching out to PyPI and avoid the ## network overhead. #name, version = spec.name, first(spec.preds)[1] #if (name, version) in self._dep_cache: #source = 'dependency cache' #return version, source version = None overrides = self.overrides.get(spec.name) ## Try the link cache, and otherwise, try PyPI if (spec.no_extra, overrides) in self._link_cache: link, version = self._link_cache[(spec.no_extra, overrides)] source = 'link cache' else: try: requirement = InstallRequirement.from_line(specline) link = self.finder.find_requirement(requirement, False) except DistributionNotFound: requirement = InstallRequirement.from_line( specline, prereleases=True) link = self.finder.find_requirement(requirement, False) link, version = self._link_hook(overrides, spec, link) # Hack to make pickle work link.comes_from = None source = 'PyPI' if link.egg_fragment: version = link.egg_fragment.rsplit('-', 1)[1] link = Link(link.url_without_fragment + "#%s=%s" % self.get_hash(link)) elif not version: _, version = splitext(link.filename)[0].rsplit('-', 1) # It's more reliable to get version from pinned spec then filename if spec.is_pinned: version = spec.pinned assert version, "Version must be set!" self._link_cache[(spec.no_extra, overrides)] = (link, version) # Take this moment to smartly insert the pinned variant of this # spec into the link_cache, too pinned_spec = Spec.from_pinned(spec.name, version) self._link_cache[pinned_spec.fullname] = (link, version) return version, source version = next((v for v in self.versions if v.name == spec.name), None) if version: spec.pinned = version.pinned specline = spec.no_extra if '==' not in specline or specline not in self._best_match_call_cache: logger.debug('- Finding best package matching %s' % spec) version = next((v for v in self.versions if v.name == spec.name), None) with logger.indent(): version, source = _find_cached_match(spec) if '==' not in specline or specline not in self._best_match_call_cache: logger.debug(' Found best match: %s (from %s)' % (version, source)) self._best_match_call_cache[specline] = True return version def get_dependencies(self, name, version, extra=()): """Gets list of dependencies from package""" spec = Spec.from_pinned(name, version, extra=extra) overrides = self.overrides.get(spec.name) extra = self.extra + extra if spec not in self._dep_call_cache: logger.debug('- Getting dependencies for %s-%s' % (name, version)) with logger.indent(): deps = self._dep_cache.get((spec, overrides)) links = self._dep_cache.get((spec, overrides, "links")) if deps is not None and links is not None: source = 'dependency cache' else: package = self.get_package(spec) deps = package.get_deps(extra=extra) deps = self._dependency_hook(overrides, spec, deps, package) self._dep_cache[(spec, overrides)] = deps links = package.get_dependency_links() self._dep_cache[(spec, overrides, "links")] = links source = 'package archive' # Run spec hook deps = [(self._spec_hook(self.overrides.get(dep.name), dep), src) if self.overrides.get(dep.name) else (dep, src) for dep, src in deps] if spec not in self._dep_call_cache: logger.debug(' Found: %s (from %s)' % (deps, source)) # At this point do not forget to add dependency links self.finder.add_dependency_links(links) self._dep_call_cache[spec] = True return deps def get_pkg_info(self, name, version): spec = Spec.from_pinned(name, version) if spec.no_extra not in self._pkg_info_call_cache: logger.debug('- Getting pkginfo for %s-%s' % (name, version)) with logger.indent(): pkg_info = self._pkg_info_cache.get(spec.no_extra) if pkg_info is not None: source = 'pkg_info cache' else: package = self.get_package(spec) pkg_info = package.get_pkginfo() pkg_info["has_tests"] = package.has_tests() self._pkg_info_cache[spec.no_extra] = pkg_info source = 'package archive' if spec.no_extra not in self._pkg_info_call_cache: logger.debug(' Found pkg_info (from %s)' % (source, )) self._pkg_info_call_cache[spec.no_extra] = True return pkg_info def get_link(self, name, version): logger.debug('- Getting link for %s-%s' % (name, version)) spec = Spec.from_pinned(name, version) self.find_best_match(spec) return self._link_cache[spec.fullname] def get_hash(self, link): if link.hash and link.hash_name: return (link.hash_name, link.hash) def md5hash(path): return ("md5", hashlib.md5(open(path, 'rb').read()).hexdigest()) url = link.url_without_fragment logger.info('- Hashing package on url %s' % (url, )) with logger.indent(): fullpath = self._get_local_package_path(link.url_without_fragment) if os.path.exists(fullpath): logger.info(' Archive cache hit: {0}'.format(link.filename)) return md5hash(fullpath) return md5hash(self._download_package(link)) def get_package(self, spec): path = self._get_or_download_package(spec.fullname) return Package(package_dir=self._extract(path), exe=self.exe, python_path=self.python_path) # Helper methods def _get_local_package_path(self, url): # noqa """Returns the full local path name for a given URL. This does not require the package archive to exist locally. In fact, this can be used to calculate the destination path for a download. """ cache_key = quote(url, '') fullpath = os.path.join(self.download_cache_root, cache_key) return fullpath def _get_or_download_package(self, specline): """Returns the local path from the package cache, downloading as needed. """ logger.debug('- Getting package location for %s' % (specline, )) with logger.indent(): link, version = self._link_cache[specline] fullpath = self._get_local_package_path(link.url_without_fragment) if os.path.exists(fullpath): logger.info(' Archive cache hit: {0}'.format(link.filename)) return fullpath logger.info(' Archive cache miss, downloading {0}...'.format( link.filename)) return self._download_package(link) def _download_package(self, link): """Downloads the given package link contents to the local package cache. Overwrites anything that's in the cache already. """ url = link.url_without_fragment logger.info('- Downloading package from %s' % (url, )) with logger.indent(): fullpath = self._get_local_package_path(url) response = _get_response_from_url(url, link) _download_url(response, link, fullpath) return fullpath def _unpack_archive(self, path, target_directory): logger.debug('- Unpacking %s' % (path, )) with logger.indent(): if path.endswith('.zip'): archive = zipfile.ZipFile(path) else: archive = tarfile.open(path) try: archive.extractall(target_directory) except IOError: logger.error("Error extracting %s" % (path, )) raise finally: archive.close() def _extract(self, path): if path in self._extract_cache: return self._extract_cache[path] logger.info('- Extracting package %s' % (path, )) build_dir = tempfile.mkdtemp() atexit.register(shutil.rmtree, build_dir) unpack_dir = os.path.join(build_dir, 'build') self._unpack_archive(path, unpack_dir) # Cache unpack self._extract_cache[path] = unpack_dir return unpack_dir
class PackageManager(object): """Interface to packages.""" def __init__( self, overrides={}, versions=[], extra=(), dependency_links=[], exe=sys.executable, python_path="", download_cache_root="", cache=None, link_hook=lambda overrides, spec, link: (link, None), dependency_hook=lambda overrides, spec, deps, package: deps, spec_hook=lambda overrides, spec: spec ): self.extra = extra self.exe, self.python_path = exe, python_path self.overrides = overrides or {} self.versions = versions or [] self._dependency_hook = dependency_hook self._link_hook = link_hook self._spec_hook = spec_hook self.finder = PackageFinder( find_links=[], index_urls=['https://pypi.python.org/simple/'], use_mirrors=True, mirrors=[], allow_all_external=True, allow_all_insecure=True ) self.finder.add_dependency_links(dependency_links) self.download_cache_root = download_cache_root cache = cache or defaultdict(dict) self._link_cache = cache["link_cache"] self._dep_cache = cache["dep_cache"] self._pkg_info_cache = cache["pkg_info_cache"] self._extract_cache = cache["extract_cache"] self._best_match_call_cache = {} self._dep_call_cache = {} self._pkg_info_call_cache = {} def find_best_match(self, spec): def _find_cached_match(spec): #if spec.is_pinned: ## If this is a pinned spec, we can take a shortcut: if it is ## found in the dependency cache, we can safely assume it has ## been downloaded before, and thus must exist. We can know ## this without every reaching out to PyPI and avoid the ## network overhead. #name, version = spec.name, first(spec.preds)[1] #if (name, version) in self._dep_cache: #source = 'dependency cache' #return version, source version = None overrides = self.overrides.get(spec.name) ## Try the link cache, and otherwise, try PyPI if (spec.no_extra, overrides) in self._link_cache: link, version = self._link_cache[(spec.no_extra, overrides)] source = 'link cache' else: try: requirement = InstallRequirement.from_line(specline) link = self.finder.find_requirement(requirement, False) except DistributionNotFound: requirement = InstallRequirement.from_line( specline, prereleases=True) link = self.finder.find_requirement(requirement, False) link, version = self._link_hook(overrides, spec, link) # Hack to make pickle work link.comes_from = None source = 'PyPI' if link.egg_fragment: version = link.egg_fragment.rsplit('-', 1)[1] link = Link( link.url_without_fragment + "#%s=%s" % self.get_hash(link) ) elif not version: _, version = splitext(link.filename)[0].rsplit('-', 1) # It's more reliable to get version from pinned spec then filename if spec.is_pinned: version = spec.pinned assert version, "Version must be set!" self._link_cache[(spec.no_extra, overrides)] = (link, version) # Take this moment to smartly insert the pinned variant of this # spec into the link_cache, too pinned_spec = Spec.from_pinned(spec.name, version) self._link_cache[pinned_spec.fullname] = (link, version) return version, source version = next((v for v in self.versions if v.name == spec.name), None) if version: spec.pinned = version.pinned specline = spec.no_extra if '==' not in specline or specline not in self._best_match_call_cache: logger.debug('- Finding best package matching %s' % spec) version = next((v for v in self.versions if v.name == spec.name), None) with logger.indent(): version, source = _find_cached_match(spec) if '==' not in specline or specline not in self._best_match_call_cache: logger.debug(' Found best match: %s (from %s)' % (version, source)) self._best_match_call_cache[specline] = True return version def get_dependencies(self, name, version, extra=()): """Gets list of dependencies from package""" spec = Spec.from_pinned(name, version, extra=extra) overrides = self.overrides.get(spec.name) extra = self.extra + extra if spec not in self._dep_call_cache: logger.debug('- Getting dependencies for %s-%s' % (name, version)) with logger.indent(): deps = self._dep_cache.get((spec, overrides)) links = self._dep_cache.get((spec, overrides, "links")) if deps is not None and links is not None: source = 'dependency cache' else: package = self.get_package(spec) deps = package.get_deps(extra=extra) deps = self._dependency_hook(overrides, spec, deps, package) self._dep_cache[(spec, overrides)] = deps links = package.get_dependency_links() self._dep_cache[(spec, overrides, "links")] = links source = 'package archive' # Run spec hook deps = [ (self._spec_hook(self.overrides.get(dep.name), dep), src) if self.overrides.get(dep.name) else (dep, src) for dep, src in deps ] if spec not in self._dep_call_cache: logger.debug(' Found: %s (from %s)' % (deps, source)) # At this point do not forget to add dependency links self.finder.add_dependency_links(links) self._dep_call_cache[spec] = True return deps def get_pkg_info(self, name, version): spec = Spec.from_pinned(name, version) if spec.no_extra not in self._pkg_info_call_cache: logger.debug('- Getting pkginfo for %s-%s' % (name, version)) with logger.indent(): pkg_info = self._pkg_info_cache.get(spec.no_extra) if pkg_info is not None: source = 'pkg_info cache' else: package = self.get_package(spec) pkg_info = package.get_pkginfo() pkg_info["has_tests"] = package.has_tests() self._pkg_info_cache[spec.no_extra] = pkg_info source = 'package archive' if spec.no_extra not in self._pkg_info_call_cache: logger.debug(' Found pkg_info (from %s)' % (source,)) self._pkg_info_call_cache[spec.no_extra] = True return pkg_info def get_link(self, name, version): logger.debug('- Getting link for %s-%s' % (name, version)) spec = Spec.from_pinned(name, version) self.find_best_match(spec) return self._link_cache[spec.fullname] def get_hash(self, link): if link.hash and link.hash_name: return (link.hash_name, link.hash) def md5hash(path): return ("md5", hashlib.md5(open(path, 'rb').read()).hexdigest()) url = link.url_without_fragment logger.info('- Hashing package on url %s' % (url,)) with logger.indent(): fullpath = self._get_local_package_path(link.url_without_fragment) if os.path.exists(fullpath): logger.info(' Archive cache hit: {0}'.format(link.filename)) return md5hash(fullpath) return md5hash(self._download_package(link)) def get_package(self, spec): path = self._get_or_download_package(spec.fullname) return Package( package_dir=self._extract(path), exe=self.exe, python_path=self.python_path ) # Helper methods def _get_local_package_path(self, url): # noqa """Returns the full local path name for a given URL. This does not require the package archive to exist locally. In fact, this can be used to calculate the destination path for a download. """ cache_key = quote(url, '') fullpath = os.path.join(self.download_cache_root, cache_key) return fullpath def _get_or_download_package(self, specline): """Returns the local path from the package cache, downloading as needed. """ logger.debug('- Getting package location for %s' % (specline,)) with logger.indent(): link, version = self._link_cache[specline] fullpath = self._get_local_package_path(link.url_without_fragment) if os.path.exists(fullpath): logger.info(' Archive cache hit: {0}'.format(link.filename)) return fullpath logger.info(' Archive cache miss, downloading {0}...'.format( link.filename )) return self._download_package(link) def _download_package(self, link): """Downloads the given package link contents to the local package cache. Overwrites anything that's in the cache already. """ url = link.url_without_fragment logger.info('- Downloading package from %s' % (url,)) with logger.indent(): fullpath = self._get_local_package_path(url) response = _get_response_from_url(url, link) _download_url(response, link, fullpath) return fullpath def _unpack_archive(self, path, target_directory): logger.debug('- Unpacking %s' % (path,)) with logger.indent(): if path.endswith('.zip'): archive = zipfile.ZipFile(path) else: archive = tarfile.open(path) try: archive.extractall(target_directory) except IOError: logger.error("Error extracting %s" % (path,)) raise finally: archive.close() def _extract(self, path): if path in self._extract_cache: return self._extract_cache[path] logger.info('- Extracting package %s' % (path,)) build_dir = tempfile.mkdtemp() atexit.register(shutil.rmtree, build_dir) unpack_dir = os.path.join(build_dir, 'build') self._unpack_archive(path, unpack_dir) # Cache unpack self._extract_cache[path] = unpack_dir return unpack_dir