def test_releasefile_md5_matching_and_ordering(self): """ check that md5-links win over non-md5 links anywhere. And otherwise the links from the index page win over scraped ones. """ result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/py-1.4.11.zip">qwe</a> <a href="../../pkg/py-1.4.10.zip#md5=2222">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 3 assert len(result.crawllinks) == 2 result.parse_index(URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.4.12.zip" /> <a href="http://pylib.org/py-1.4.11.zip#md5=1111" /> <a href="http://pylib.org/py-1.4.10.zip#md5=12ab" /> """, scrape=False) assert len(result.crawllinks) == 2 assert len(result.releaselinks) == 3 link1, link2, link3 = result.releaselinks assert link1.url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert link2.url == "http://pylib.org/py-1.4.11.zip#md5=1111" assert link3.url == "https://pypi.org/pkg/py-1.4.10.zip#md5=2222"
def test_parse_index_two_eggs_same_url(self): simplepy = URL("https://pypi.org/simple/Py/") result = parse_index( simplepy, """<a href="../../pkg/pyzip#egg=py-dev">qwe2</a> <a href="../../pkg/pyzip#egg=py-dev">qwe</a> """) assert len(result.releaselinks) == 1
def test_parse_index_simple_dir_egg_issue63(self): simplepy = URL("https://pypi.org/simple/py/") result = parse_index( simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/#egg=py-dev">qwe</a> """) assert len(result.releaselinks) == 1
def test_redownload_locally_removed_release(mapp, simpypi): from devpi_common.url import URL mapp.create_and_login_user('mirror') indexconfig = dict(type="mirror", mirror_url=simpypi.simpleurl, mirror_cache_expiry=0) mapp.create_index("mirror", indexconfig=indexconfig) mapp.use("mirror/mirror") content = b'14' simpypi.add_release('pkg', pkgver='pkg-1.0.zip') simpypi.add_file('/pkg/pkg-1.0.zip', content) result = mapp.getreleaseslist("pkg") file_relpath = '+files' + URL(result[0]).path assert len(result) == 1 r = mapp.downloadrelease(200, result[0]) assert r == content with mapp.xom.keyfs.transaction(write=False) as tx: assert tx.conn.io_file_exists(file_relpath) # now remove the local copy with mapp.xom.keyfs.transaction(write=True) as tx: tx.conn.io_file_delete(file_relpath) with mapp.xom.keyfs.transaction(write=False) as tx: assert not tx.conn.io_file_exists(file_relpath) serial = mapp.xom.keyfs.get_current_serial() # and download again r = mapp.downloadrelease(200, result[0]) assert r == content with mapp.xom.keyfs.transaction(write=False) as tx: assert tx.conn.io_file_exists(file_relpath) # the serial should not have increased assert serial == mapp.xom.keyfs.get_current_serial()
def test_parse_index_simple_nocase(self): simplepy = URL("https://pypi.org/simple/Py/") result = parse_index( simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/PY-1.4.13.zip">qwe</a> """) assert len(result.releaselinks) == 2
def test_parse_index_normalized_name(self): simplepy = URL("https://pypi.org/simple/ndg-httpsclient/") result = parse_index(simplepy, """ <a href="../../pkg/ndg_httpsclient-1.0.tar.gz" /> """) assert len(result.releaselinks) == 1 assert result.releaselinks[0].url.endswith("ndg_httpsclient-1.0.tar.gz")
def test_releasefile_and_scrape(self): result = parse_index(self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 1 result.parse_index(URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.1-py27.egg" /> <a href="http://pylib.org/other" rel="download" /> """) assert len(result.releaselinks) == 2 links = list(result.releaselinks) assert links[0].url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert links[1].url == "http://pylib.org/py-1.1-py27.egg"
def test_parse_index_egg_svnurl(self, monkeypatch): # strange case reported by fschulze/witsch where # urlparsing will yield a fragment for svn urls. # it's not exactly clear how urlparse.uses_fragment # sometimes contains "svn" but it's good to check # that we are not sensitive to the issue. try: import urllib.parse as urlparse except ImportError: # PY2 import urlparse monkeypatch.setattr(urlparse, "uses_fragment", urlparse.uses_fragment + ["svn"]) simplepy = URL("https://pypi.org/simple/zope.sqlalchemy/") result = parse_index( simplepy, '<a href="svn://svn.zope.org/repos/main/' 'zope.sqlalchemy/trunk#egg=zope.sqlalchemy-dev" />') assert len(result.releaselinks) == 0 assert len(result.egglinks) == 0
def test_parse_index_unparseable_url(self): simple = URL("https://pypi.org/simple/x123/") result = parse_index(simple, '<a href="http:" />') assert len(result.releaselinks) == 0
def test_parse_index_with_num_in_project(self): simple = URL("https://pypi.org/simple/py-4chan/") result = parse_index(simple, '<a href="pkg/py-4chan-1.0.zip"/>') assert len(result.releaselinks) == 1 assert result.releaselinks[0].basename == "py-4chan-1.0.zip"
def test_parse_index_invalid_link(self, pypistage): result = parse_index( self.simplepy, ''' <a rel="download" href="https:/host.com/123" /> ''') assert result.crawllinks == {URL('https://pypi.org/host.com/123')}
class TestIndexParsing: simplepy = URL("https://pypi.org/simple/py/") @pytest.mark.parametrize("hash_type,hash_value", [("sha256", "090123"), ("sha224", "1209380123"), ("md5", "102938")]) def test_parse_index_simple_hash_types(self, hash_type, hash_value): result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#%s=%s" /a>""" % (hash_type, hash_value)) link, = result.releaselinks assert link.basename == "py-1.4.12.zip" assert link.hash_spec == "%s=%s" % (hash_type, hash_value) if hash_type == "md5": assert link.md5 == hash_value else: assert link.md5 is None assert link.hash_algo == getattr(hashlib, hash_type) def test_parse_index_simple_tilde(self): result = parse_index( self.simplepy, """<a href="/~user/py-1.4.12.zip#md5=12ab">qwe</a>""") link, = result.releaselinks assert link.basename == "py-1.4.12.zip" assert link.url.endswith("/~user/py-1.4.12.zip#md5=12ab") def test_parse_index_simple_nocase(self): simplepy = URL("https://pypi.org/simple/Py/") result = parse_index( simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/PY-1.4.13.zip">qwe</a> <a href="../../pkg/pyzip#egg=py-dev">qwe</a> """) assert len(result.releaselinks) == 3 def test_parse_index_simple_dir_egg_issue63(self): simplepy = URL("https://pypi.org/simple/py/") result = parse_index( simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/#egg=py-dev">qwe</a> """) assert len(result.releaselinks) == 1 def test_parse_index_egg_svnurl(self, monkeypatch): # strange case reported by fschulze/witsch where # urlparsing will yield a fragment for svn urls. # it's not exactly clear how urlparse.uses_fragment # sometimes contains "svn" but it's good to check # that we are not sensitive to the issue. try: import urllib.parse as urlparse except ImportError: # PY2 import urlparse monkeypatch.setattr(urlparse, "uses_fragment", urlparse.uses_fragment + ["svn"]) simplepy = URL("https://pypi.org/simple/zope.sqlalchemy/") result = parse_index( simplepy, '<a href="svn://svn.zope.org/repos/main/' 'zope.sqlalchemy/trunk#egg=zope.sqlalchemy-dev" />') assert len(result.releaselinks) == 0 assert len(result.egglinks) == 0 #assert 0, (result.releaselinks, result.egglinks) def test_parse_index_normalized_name(self): simplepy = URL("https://pypi.org/simple/ndg-httpsclient/") result = parse_index( simplepy, """ <a href="../../pkg/ndg_httpsclient-1.0.tar.gz" /> """) assert len(result.releaselinks) == 1 assert result.releaselinks[0].url.endswith( "ndg_httpsclient-1.0.tar.gz") def test_parse_index_two_eggs_same_url(self): simplepy = URL("https://pypi.org/simple/Py/") result = parse_index( simplepy, """<a href="../../pkg/pyzip#egg=py-dev">qwe2</a> <a href="../../pkg/pyzip#egg=py-dev">qwe</a> """) assert len(result.releaselinks) == 1 def test_parse_index_simple_nomatch(self): result = parse_index(self.simplepy, """<a href="../../pkg/py-1.3.html">qwe</a>""") assert not result.releaselinks @pytest.mark.parametrize("rel", ["homepage", "download"]) def test_parse_index_with_rel(self, rel): result = parse_index( self.simplepy, """ <a href="http://pylib.org" rel="%s">whatever</a> <a href="http://pylib2.org" rel="%s">whatever2</a> <a href="http://pylib3.org">py-1.0.zip</a> <a href="http://pylib2.org/py-1.0.zip" rel="%s">whatever2</a> """ % (rel, rel, rel)) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link == "http://pylib2.org/py-1.0.zip" assert len(result.crawllinks) == 2 assert result.crawllinks == \ set(["http://pylib.org", "http://pylib2.org"]) def test_parse_index_invalid_link(self, pypistage): result = parse_index( self.simplepy, ''' <a rel="download" href="https:/host.com/123" /> ''') assert result.crawllinks == {URL('https://pypi.org/host.com/123')} def test_parse_index_with_egg(self): result = parse_index( self.simplepy, """<a href="http://bb.org/download/py.zip#egg=py-dev" /> <a href="http://bb.org/download/py-1.0.zip" /> <a href="http://bb.org/download/py.zip#egg=something-dev" /> """) assert len(result.releaselinks) == 2 link, link2 = result.releaselinks assert link.basename == "py.zip" assert link.eggfragment == "py-dev" assert link2.basename == "py-1.0.zip" def test_parse_index_with_wheel(self): result = parse_index( self.simplepy, """<a href="pkg/py-1.0-cp27-none-linux_x86_64.whl" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0-cp27-none-linux_x86_64.whl" @pytest.mark.parametrize("basename", [ "py-1.3.1.tar.gz", "py-1.3.1-1.fc12.src.rpm", "py-docs-1.0.zip", "py-1.1.0.win-amd64.exe", "py.tar.gz", "py-0.8.msi", "py-0.10.0.dmg", "py-0.8.deb", "py-12.0.0.win32-py2.7.msi", "py-1.3.1-1.0rc4.tar.gz", "py-1.0.1.tar.bz2" ]) def test_parse_index_with_valid_basenames(self, basename): result = parse_index(self.simplepy, '<a href="pkg/%s" />' % basename) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == basename def test_parse_index_with_num_in_project(self): simple = URL("https://pypi.org/simple/py-4chan/") result = parse_index(simple, '<a href="pkg/py-4chan-1.0.zip"/>') assert len(result.releaselinks) == 1 assert result.releaselinks[0].basename == "py-4chan-1.0.zip" def test_parse_index_unparseable_url(self): simple = URL("https://pypi.org/simple/x123/") result = parse_index(simple, '<a href="http:" />') assert len(result.releaselinks) == 0 def test_parse_index_ftp_ignored_for_now(self): result = parse_index( self.simplepy, """<a href="http://bb.org/download/py-1.0.zip" /> <a href="ftp://bb.org/download/py-1.0.tar.gz" /> <a rel="download" href="ftp://bb.org/download/py-1.1.tar.gz" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0.zip" def test_parse_index_with_two_eggs_ordering(self): # it seems that pip/easy_install in some cases # rely on the exact ordering of eggs in the html page # for example with nose, there are two eggs and the second/last # one is chosen due to the internal pip/easy_install algorithm result = parse_index( self.simplepy, """<a href="http://bb.org/download/py.zip#egg=py-dev" /> <a href="http://other/master#egg=py-dev" /> """) assert len(result.releaselinks) == 2 link1, link2 = result.releaselinks assert link1.basename == "master" assert link1.eggfragment == "py-dev" assert link2.basename == "py.zip" assert link2.eggfragment == "py-dev" def test_parse_index_with_matchingproject_no_version(self): result = parse_index( self.simplepy, """<a href="http://bb.org/download/p.zip" /> <a href="http://bb.org/download/py-1.0.zip" />""") assert len(result.releaselinks) == 1 def test_parse_index_with_non_parseable_hrefs(self): result = parse_index( self.simplepy, """<a href="qlkwje 1lk23j123123" /> <a href="http://bb.org/download/py-1.0.zip" />""") assert len(result.releaselinks) == 1 def test_releasefile_and_scrape(self): result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 1 assert len(result.crawllinks) == 2 result.parse_index(URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.1-py27.egg" /> <a href="http://pylib.org/other" rel="download" /> """, scrape=False) assert len(result.crawllinks) == 2 assert len(result.releaselinks) == 2 links = list(result.releaselinks) assert links[0].url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert links[1].url == "http://pylib.org/py-1.1-py27.egg" def test_releasefile_and_scrape_no_ftp(self): result = parse_index( self.simplepy, """<a href="ftp://pylib2.org/py-1.0.tar.gz" rel="download">whatever2</a> """) assert len(result.releaselinks) == 0 assert len(result.crawllinks) == 0 def test_releasefile_md5_matching_and_ordering(self): """ check that md5-links win over non-md5 links anywhere. And otherwise the links from the index page win over scraped ones. """ result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/py-1.4.11.zip">qwe</a> <a href="../../pkg/py-1.4.10.zip#md5=2222">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 3 assert len(result.crawllinks) == 2 result.parse_index(URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.4.12.zip" /> <a href="http://pylib.org/py-1.4.11.zip#md5=1111" /> <a href="http://pylib.org/py-1.4.10.zip#md5=12ab" /> """, scrape=False) assert len(result.crawllinks) == 2 assert len(result.releaselinks) == 3 link1, link2, link3 = result.releaselinks assert link1.url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert link2.url == "http://pylib.org/py-1.4.11.zip#md5=1111" assert link3.url == "https://pypi.org/pkg/py-1.4.10.zip#md5=2222"
class TestIndexParsing: simplepy = URL("https://pypi.org/simple/py/") @pytest.mark.parametrize("hash_type,hash_value", [("sha256", "090123"), ("sha224", "1209380123"), ("md5", "102938")]) def test_parse_index_simple_hash_types(self, hash_type, hash_value): result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#%s=%s" /a>""" % (hash_type, hash_value)) link, = result.releaselinks assert link.basename == "py-1.4.12.zip" assert link.hash_spec == "%s=%s" % (hash_type, hash_value) if hash_type == "md5": assert link.md5 == hash_value else: assert link.md5 is None assert link.hash_algo == getattr(hashlib, hash_type) def test_parse_index_simple_tilde(self): result = parse_index( self.simplepy, """<a href="/~user/py-1.4.12.zip#md5=12ab">qwe</a>""") link, = result.releaselinks assert link.basename == "py-1.4.12.zip" assert link.url.endswith("/~user/py-1.4.12.zip#md5=12ab") def test_parse_index_simple_nocase(self): simplepy = URL("https://pypi.org/simple/Py/") result = parse_index( simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/PY-1.4.13.zip">qwe</a> """) assert len(result.releaselinks) == 2 def test_parse_index_normalized_name(self): simplepy = URL("https://pypi.org/simple/ndg-httpsclient/") result = parse_index( simplepy, """ <a href="../../pkg/ndg_httpsclient-1.0.tar.gz" /> """) assert len(result.releaselinks) == 1 assert result.releaselinks[0].url.endswith( "ndg_httpsclient-1.0.tar.gz") def test_parse_index_simple_nomatch(self): result = parse_index(self.simplepy, """<a href="../../pkg/py-1.3.html">qwe</a>""") assert not result.releaselinks @pytest.mark.parametrize("rel", ["homepage", "download"]) def test_parse_index_with_rel(self, rel): result = parse_index( self.simplepy, """ <a href="http://pylib.org" rel="%s">whatever</a> <a href="http://pylib2.org" rel="%s">whatever2</a> <a href="http://pylib3.org">py-1.0.zip</a> <a href="http://pylib2.org/py-1.0.zip" rel="%s">whatever2</a> """ % (rel, rel, rel)) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link == "http://pylib2.org/py-1.0.zip" def test_parse_index_invalid_link(self, pypistage): result = parse_index( self.simplepy, ''' <a rel="download" href="https:/host.com/123" /> ''') assert result.releaselinks == [] def test_parse_index_with_wheel(self): result = parse_index( self.simplepy, """<a href="pkg/py-1.0-cp27-none-linux_x86_64.whl" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0-cp27-none-linux_x86_64.whl" def test_parse_index_with_requires_python(self): result = parse_index( self.simplepy, """<a href="pkg/py-1.0.zip" data-requires-python="<3" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0.zip" assert link.requires_python == "<3" def test_parse_index_with_requires_python_hash_spec_is_better(self): result = parse_index( self.simplepy, """<a href="pkg/py-1.0.zip" data-requires-python="<3" /> <a href="pkg/py-1.0.zip#md5=pony"/> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0.zip" assert link.hash_spec == "md5=pony" assert link.requires_python is None def test_parse_index_with_requires_python_first_with_hash_spec_kept(self): result = parse_index( self.simplepy, """<a href="pkg/py-1.0.zip#md5=pony"/> <a href="pkg/py-1.0.zip#md5=pony" data-requires-python="<3" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0.zip" assert link.hash_spec == "md5=pony" assert link.requires_python is None @pytest.mark.parametrize("basename", [ "py-1.3.1.tar.gz", "py-1.3.1-1.fc12.src.rpm", "py-docs-1.0.zip", "py-1.1.0.win-amd64.exe", "py.tar.gz", "py-0.8.msi", "py-0.10.0.dmg", "py-0.8.deb", "py-12.0.0.win32-py2.7.msi", "py-1.3.1-1.0rc4.tar.gz", "py-1.0.1.tar.bz2" ]) def test_parse_index_with_valid_basenames(self, basename): result = parse_index(self.simplepy, '<a href="pkg/%s" />' % basename) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == basename def test_parse_index_with_num_in_project(self): simple = URL("https://pypi.org/simple/py-4chan/") result = parse_index(simple, '<a href="pkg/py-4chan-1.0.zip"/>') assert len(result.releaselinks) == 1 assert result.releaselinks[0].basename == "py-4chan-1.0.zip" def test_parse_index_unparseable_url(self): simple = URL("https://pypi.org/simple/x123/") result = parse_index(simple, '<a href="http:" />') assert len(result.releaselinks) == 0 def test_parse_index_ftp_ignored_for_now(self): result = parse_index( self.simplepy, """<a href="http://bb.org/download/py-1.0.zip" /> <a href="ftp://bb.org/download/py-1.0.tar.gz" /> <a rel="download" href="ftp://bb.org/download/py-1.1.tar.gz" /> """) assert len(result.releaselinks) == 1 link, = result.releaselinks assert link.basename == "py-1.0.zip" def test_parse_index_with_matchingproject_no_version(self): result = parse_index( self.simplepy, """<a href="http://bb.org/download/p.zip" /> <a href="http://bb.org/download/py-1.0.zip" />""") assert len(result.releaselinks) == 1 def test_parse_index_with_non_parseable_hrefs(self): result = parse_index( self.simplepy, """<a href="qlkwje 1lk23j123123" /> <a href="http://bb.org/download/py-1.0.zip" />""") assert len(result.releaselinks) == 1 def test_releasefile_and_scrape(self): result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 1 result.parse_index( URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.1-py27.egg" /> <a href="http://pylib.org/other" rel="download" /> """) assert len(result.releaselinks) == 2 links = list(result.releaselinks) assert links[0].url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert links[1].url == "http://pylib.org/py-1.1-py27.egg" def test_releasefile_and_scrape_no_ftp(self): result = parse_index( self.simplepy, """<a href="ftp://pylib2.org/py-1.0.tar.gz" rel="download">whatever2</a> """) assert len(result.releaselinks) == 0 def test_releasefile_md5_matching_and_ordering(self): """ check that md5-links win over non-md5 links anywhere. And otherwise the links from the index page win over scraped ones. """ result = parse_index( self.simplepy, """<a href="../../pkg/py-1.4.12.zip#md5=12ab">qwe</a> <a href="../../pkg/py-1.4.11.zip">qwe</a> <a href="../../pkg/py-1.4.10.zip#md5=2222">qwe</a> <a href="http://pylib.org" rel="homepage">whatever</a> <a href="http://pylib2.org" rel="download">whatever2</a> """) assert len(result.releaselinks) == 3 result.parse_index( URL("http://pylib.org"), """ <a href="http://pylib.org/py-1.4.12.zip" /> <a href="http://pylib.org/py-1.4.11.zip#md5=1111" /> <a href="http://pylib.org/py-1.4.10.zip#md5=12ab" /> """) assert len(result.releaselinks) == 3 link1, link2, link3 = result.releaselinks assert link1.url == "https://pypi.org/pkg/py-1.4.12.zip#md5=12ab" assert link2.url == "http://pylib.org/py-1.4.11.zip#md5=1111" assert link3.url == "https://pypi.org/pkg/py-1.4.10.zip#md5=2222"