def get_location(self, dist, dependency_links): for url in dependency_links: egg_fragment = Link(url).egg_fragment if not egg_fragment: continue if '-' in egg_fragment: ## FIXME: will this work when a package has - in the name? key = '-'.join(egg_fragment.split('-')[:-1]).lower() else: key = egg_fragment if key == dist.key: return url.split('#', 1)[0] return None
def test_unpack_http_url_with_urllib_response_without_content_type(): """ It should download and unpack files even if no Content-Type header exists """ def _get_response_from_url_mock(*args, **kw): resp = _get_response_from_url_original(*args, **kw) del resp.info()['content-type'] return resp with patch('pip.download._get_response_from_url', _get_response_from_url_mock) as mocked: uri = path_to_url2( os.path.join(tests_data, 'packages', 'simple-1.0.tar.gz')) link = Link(uri) temp_dir = mkdtemp() try: unpack_http_url(link, temp_dir, download_cache=None, download_dir=None) assert set(os.listdir(temp_dir)) == set([ 'PKG-INFO', 'setup.cfg', 'setup.py', 'simple', 'simple.egg-info' ]) finally: rmtree(temp_dir)
def test_unpack_http_url_bad_cache_checksum(mock_get_response, mock_unpack_file): """ If cached download has bad checksum, re-download. """ base_url = 'http://www.example.com/somepackage.tgz' contents = b('downloaded') download_hash = hashlib.new('sha1', contents) link = Link(base_url + '#sha1=' + download_hash.hexdigest()) response = mock_get_response.return_value = MockResponse(contents) response.info = lambda: {'content-type': 'application/x-tar'} response.geturl = lambda: base_url cache_dir = mkdtemp() try: cache_file = os.path.join(cache_dir, urllib.quote(base_url, '')) cache_ct_file = cache_file + '.content-type' _write_file(cache_file, 'some contents') _write_file(cache_ct_file, 'application/x-tar') unpack_http_url(link, 'location', download_cache=cache_dir) # despite existence of cached file with bad hash, downloaded again mock_get_response.assert_called_once_with(base_url, link) # cached file is replaced with newly downloaded file with open(cache_file) as fh: assert fh.read() == 'downloaded' finally: rmtree(cache_dir)
def _link(self): try: return self._req.link except AttributeError: # The link attribute isn't available prior to pip 6.1.0, so fall # back to the now deprecated 'url' attribute. return Link(self._req.url) if self._req.url else None
def test_updates_package_in_nested_requirements(self): tempdir = tempfile.mkdtemp() requirements = os.path.join(tempdir, 'requirements-with-nested-reqfile.txt') requirements_nested = os.path.join(tempdir, 'requirements-nested.txt') shutil.copy('tests/samples/requirements-with-nested-reqfile.txt', requirements) shutil.copy('tests/samples/requirements-nested.txt', requirements_nested) args = ['-r', requirements] with utils.mock.patch('pip.index.PackageFinder.find_all_candidates' ) as mock_find_all_candidates: project = 'readtime' version = '0.10.1' link = Link('') candidate = InstallationCandidate(project, version, link) mock_find_all_candidates.return_value = [candidate] result = self.runner.invoke(pur, args) self.assertIsNone(result.exception) expected_output = "Updated readtime: 0.9 -> 0.10.1\nAll requirements up-to-date.\n" self.assertEquals(u(result.output), u(expected_output)) self.assertEquals(result.exit_code, 0) expected_requirements = open( 'tests/samples/results/test_updates_package_in_nested_requirements' ).read() self.assertEquals(open(requirements).read(), expected_requirements) expected_requirements = open( 'tests/samples/results/test_updates_package_in_nested_requirements_nested' ).read() self.assertEquals( open(requirements_nested).read(), expected_requirements)
def test_updates_from_alt_index_url(self): requirements = 'tests/samples/requirements-with-alt-index-url.txt' tempdir = tempfile.mkdtemp() tmpfile = os.path.join(tempdir, 'requirements.txt') shutil.copy(requirements, tmpfile) args = ['-r', tmpfile] class PackageFinderSpy(PackageFinder): _spy = None def __init__(self, *args, **kwargs): super(PackageFinderSpy, self).__init__(*args, **kwargs) PackageFinderSpy._spy = self with utils.mock.patch('pur.PackageFinder', wraps=PackageFinderSpy) as mock_finder: with utils.mock.patch('pip.index.PackageFinder.find_all_candidates' ) as mock_find_all_candidates: project = 'flask' version = '12.1' link = Link('') candidate = InstallationCandidate(project, version, link) mock_find_all_candidates.return_value = [candidate] self.runner.invoke(pur, args) self.assertTrue(mock_finder.called) self.assertEqual( PackageFinderSpy._spy.index_urls, ['http://pypi.example.com', 'https://pypi.example2.com']) self.assertEqual(PackageFinderSpy._spy.secure_origins, [('*', 'pypi.example.com', '*')])
def from_editable(cls, editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pip.index import Link name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) else: req = None return cls( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def from_editable(cls, editable_req, comes_from=None, default_vcs=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pip.index import Link name, url, extras_override, editable_options = parse_editable( editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None res = cls(name, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, editable_options=editable_options, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache) if extras_override is not None: res.extras = extras_override return res
def test_unpack_file_url_thats_a_dir(self, tmpdir, data): self.prep(tmpdir, data) dist_path = data.packages.join("FSPkg") dist_url = Link(path_to_url(dist_path)) unpack_file_url(dist_url, self.build_dir, download_dir=self.download_dir) assert os.path.isdir(os.path.join(self.build_dir, 'fspkg'))
def test_unpack_http_url_with_urllib_response_without_content_type(data): """ It should download and unpack files even if no Content-Type header exists """ _real_session = PipSession() def _fake_session_get(*args, **kwargs): resp = _real_session.get(*args, **kwargs) del resp.headers["Content-Type"] return resp session = Mock() session.get = _fake_session_get uri = path_to_url(data.packages.join("simple-1.0.tar.gz")) link = Link(uri) temp_dir = mkdtemp() try: unpack_http_url( link, temp_dir, download_dir=None, session=session, ) assert set(os.listdir(temp_dir)) == set([ 'PKG-INFO', 'setup.cfg', 'setup.py', 'simple', 'simple.egg-info' ]) finally: rmtree(temp_dir)
def test_check_hash_sha512_invalid(): file_path = os.path.join(packages, "gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz#sha512=deadbeef") download_hash = _get_hash_from_file(file_path, file_link) assert_raises(InstallationError, _check_hash, download_hash, file_link)
def faster_find_requirement(self, req, upgrade): """see faster_pip_packagefinder""" from pip.index import BestVersionAlreadyInstalled if req_is_absolute(req.req): # if the version is pinned-down by a == # first try to use any installed package that satisfies the req if req.satisfied_by: if upgrade: # as a matter of api, find_requirement() only raises during upgrade -- shrug raise BestVersionAlreadyInstalled else: return None # then try an optimistic search for a .whl file: from os.path import join from glob import glob from pip.wheel import Wheel from pip.index import Link for findlink in self.find_links: if findlink.startswith('file://'): findlink = findlink[7:] else: continue # this matches the name-munging done in pip.wheel: reqname = req.name.replace('-', '_') for link in glob(join(findlink, reqname + '-*.whl')): link = Link('file://' + link) wheel = Wheel(link.filename) if wheel.version in req.req and wheel.supported(): return link # otherwise, do the full network search return self.unpatched['find_requirement'](self, req, upgrade)
def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, update=True, pycompile=True, markers=None, isolated=False, options=None, wheel_cache=None, constraint=False, extras=()): assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint if source_dir is not None: self.source_dir = os.path.normpath(os.path.abspath(source_dir)) else: self.source_dir = None self.editable = editable self._wheel_cache = wheel_cache if link is not None: self.link = self.original_link = link else: from pip.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: self.extras = extras elif req: self.extras = set( pkg_resources.safe_extra(extra) for extra in req.extras ) else: self.extras = set() if markers is not None: self.markers = markers else: self.markers = req and req.marker self._egg_info_path = None # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None # This hold the pkg_resources.Distribution object if this requirement # conflicts with another installed distribution: self.conflicts_with = None # Temporary build location self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. self._ideal_build_dir = None # True if the editable should be updated: self.update = update # Set to True after successful installation self.install_succeeded = None # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.use_user_site = False self.target_dir = None self.options = options if options else {} self.pycompile = pycompile # Set to True after successful preparation of this requirement self.prepared = False self.isolated = isolated
def test_check_hash_md5_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz" "#md5=d41d8cd98f00b204e9800998ecf8427e") download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def test_download_url(self, mockget): mockget.return_value = Mock() mockget.return_value.content = self.dists['dp'].bytes() rd = self.makeone() pinfo, outfile = rd.download_url( Link('http://fakeurl.com/dummypackage.tar.gz#egg=dummypackage')) assert outfile.exists() assert pinfo.name == 'dummypackage'
def test_check_hash_sha1_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz" "#sha1=da39a3ee5e6b4b0d3255bfef95601890afd80709") download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def test_check_hash_sha512_invalid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz#sha512=deadbeef") download_hash = _get_hash_from_file(file_path, file_link) with pytest.raises(InstallationError): _check_hash(download_hash, file_link)
def test_get_hash_from_file_unknown(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz" "#unknown_hash=d41d8cd98f00b204e9800998ecf8427e") download_hash = _get_hash_from_file(file_path, file_link) assert download_hash is None
def test_check_hash_sha512_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link( "http://testserver/gmpy-1.15.tar.gz#sha512=cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" ) download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def test_http_link(self): link = Link( 'https://pypi.python.org/packages/source/p/pip/pip-7.0.3.tar.gz' '#md5=e972d691ff6779ffb4b594449bac3e43') assert link_to_nix(link) == ( 'fetchurl {\n' ' url = "https://pypi.python.org/packages/source/p/pip/pip-7.0.3.tar.gz";\n' ' md5 = "e972d691ff6779ffb4b594449bac3e43";\n' '}')
def test_check_hash_sha224_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link( "http://testserver/gmpy-1.15.tar.gz#sha224=d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f'" ) download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def test_check_hash_sha384_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link( "http://testserver/gmpy-1.15.tar.gz#sha384=38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b" ) download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def test_get_hash_from_file_md5(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz" "#md5=d41d8cd98f00b204e9800998ecf8427e") download_hash = _get_hash_from_file(file_path, file_link) assert download_hash.digest_size == 16 assert download_hash.hexdigest() == "d41d8cd98f00b204e9800998ecf8427e"
def test_check_hash_sha256_valid(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link( "http://testserver/gmpy-1.15.tar.gz#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) download_hash = _get_hash_from_file(file_path, file_link) _check_hash(download_hash, file_link)
def download_url(url, download_dir, sha256=None): cmd = DownloadCommand() options, _ = cmd.parse_args([]) link = Link(url) session = cmd._build_session(options) hashes = Hashes({"sha256": [sha256]}) if sha256 else None downloaded_path = _check_download_dir(link, download_dir, hashes) if downloaded_path: return downloaded_path try: downloaded_path, _ = _download_http_url(Link(url), session, download_dir, hashes) except HashMismatch0 as e: expected = e.allowed["sha256"][0] actual = e.gots["sha256"].hexdigest() raise HashMismatch(url, expected, actual) else: return _ensure_expected_download_path(downloaded_path, link)
def test_http_link(self): link = Link( 'https://pypi.python.org/packages/source/p/pip/pip-7.0.3.tar.gz' '#md5=e972d691ff6779ffb4b594449bac3e43') assert link_to_nix(link) == ( 'fetchurl {\n' ' url = "https://pypi.python.org/packages/source/p/pip/pip-7.0.3.tar.gz";\n' ' sha256 = "1zdgl0qsgsh71b397120y7vw3rkbisrgws2rqv5c4vbgba19iidl";\n' '}')
def _link_hook(self, overrides, spec, link): overrides = overrides or {} if overrides.get("src"): logger.info( '===> Link override %s found for package %s', overrides, spec) _, version = splitext(link.filename)[0].rsplit('-', 1) spec = Spec.from_pinned(name=spec.name, version=version) src = env.from_string( overrides.get("src")).render({"spec": spec}) link = Link(src) # Hack to make pickle work link.comes_from = None return link, spec.pinned return link, None
def test_get_hash_from_file_sha1(data): file_path = data.packages.join("gmpy-1.15.tar.gz") file_link = Link("http://testserver/gmpy-1.15.tar.gz" "#sha1=da39a3ee5e6b4b0d3255bfef95601890afd80709") download_hash = _get_hash_from_file(file_path, file_link) assert download_hash.digest_size == 20 assert download_hash.hexdigest() == ( "da39a3ee5e6b4b0d3255bfef95601890afd80709")
def _downloaded_filename(self): """Download the package's archive if necessary, and return its filename. --no-deps is implied, as we have reimplemented the bits that would ordinarily do dependency resolution. """ # Peep doesn't support requirements that don't come down as a single # file, because it can't hash them. Thus, it doesn't support editable # requirements, because pip itself doesn't support editable # requirements except for "local projects or a VCS url". Nor does it # support VCS requirements yet, because we haven't yet come up with a # portable, deterministic way to hash them. In summary, all we support # is == requirements and tarballs/zips/etc. # TODO: Stop on reqs that are editable or aren't ==. finder = package_finder(self._argv) # If the requirement isn't already specified as a URL, get a URL # from an index: link = (finder.find_requirement(self._req, upgrade=False) if self._req.url is None else Link(self._req.url)) if link: lower_scheme = link.scheme.lower( ) # pip lower()s it for some reason. if lower_scheme == 'http' or lower_scheme == 'https': file_path = self._download(link) return basename(file_path) elif lower_scheme == 'file': # The following is inspired by pip's unpack_file_url(): link_path = url_to_path(link.url_without_fragment) if isdir(link_path): raise UnsupportedRequirementError( "%s: %s is a directory. So that it can compute " "a hash, peep supports only filesystem paths which " "point to files" % (self._req, link.url_without_fragment)) else: copy(link_path, self._temp_path) return basename(link_path) else: raise UnsupportedRequirementError( "%s: The download link, %s, would not result in a file " "that can be hashed. Peep supports only == requirements, " "file:// URLs pointing to files (not folders), and " "http:// and https:// URLs pointing to tarballs, zips, " "etc." % (self._req, link.url)) else: raise UnsupportedRequirementError( "%s: couldn't determine where to download this requirement from." % (self._req, ))
def test_link_package_versions_match_wheel(self): """Test that 'pytest' archives match for 'pytest'""" # TODO: Uncomment these, when #1217 is fixed # link = Link('http:/yo/pytest-1.0.tar.gz') # result = self.finder._link_package_versions(link, self.search_name) # assert result == [(self.parsed_version, link, self.version)], result link = Link('http:/yo/pytest-1.0-py2.py3-none-any.whl') result = self.finder._link_package_versions(link, self.search_name) assert result == [(self.parsed_version, link, self.version)], result
def test_link_sorting_raises_when_wheel_unsupported(self): links = [ ( parse_version('1.0'), Link('simple-1.0-py2.py3-none-TEST.whl'), '1.0', ), ] finder = PackageFinder([], [], use_wheel=True, session=PipSession()) with pytest.raises(InstallationError): finder._sort_versions(links)
def from_line( cls, name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ from pip.index import Link if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers = name.split(marker_sep, 1) markers = markers.strip() if not markers: markers = None else: markers = None name = name.strip() req = None path = os.path.normpath(os.path.abspath(name)) link = None extras = None if is_url(name): link = Link(name) else: p, extras = _strip_extras(path) if (os.path.isdir(p) and (os.path.sep in name or name.startswith('.'))): if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. File 'setup.py' " "not found." % name ) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name ) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Normalize URLs link.normalize() # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req = link.egg_fragment # a requirement specifier else: req = name options = options if options else {} res = cls(req, comes_from, link=link, markers=markers, isolated=isolated, options=options, wheel_cache=wheel_cache, constraint=constraint) if extras: res.extras = Requirement('placeholder' + extras).extras return res