async def test_keep_index_versions_stores_different_prior_versions(
mirror: Mirror, ) -> None:
simple_path = Path("web/simple/foo")
versions_path = simple_path / "versions"
mirror.packages_to_sync = {"foo": 1}
mirror.keep_index_versions = 2
with freeze_time("2018-10-27"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
mirror.packages_to_sync = {"foo": 1}
with freeze_time("2018-10-28"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
version_files = sorted(os.listdir(versions_path))
assert len(version_files) == 2
assert version_files[0].startswith("index_1_2018-10-27")
assert version_files[1].startswith("index_1_2018-10-28")
link_path = simple_path / "index.html"
assert os.path.islink(link_path)
assert os.path.basename(os.readlink(str(link_path))) == version_files[1]
def test_latest_releases_ensure_reusable(self) -> None:
"""
Tests the filter multiple times to ensure no state is preserved and
thus is reusable between packages
"""
mock_config(self.config_contents)
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg1 = Package("foo", 1)
pkg1._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"1.0.0": {},
"1.1.0": {},
"1.2.0": {},
"2.0.0": {},
},
}
pkg2 = Package("bar", 1)
pkg2._metadata = {
"info": {
"name": "bar",
"version": "0.3.0"
},
"releases": {
"0.1.0": {},
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"0.1.4": {},
"0.1.5": {},
"0.2.0": {},
"0.3.0": {},
},
}
pkg1.filter_all_releases(mirror.filters.filter_release_plugins())
pkg2.filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg1.releases == {"1.2.0": {}, "2.0.0": {}}
assert pkg2.releases == {"0.2.0": {}, "0.3.0": {}}
def test_survives_exceptions_from_record_finished_package(mirror, requests):
def record_finished_package(name):
import errno
raise IOError(errno.EBADF, "Some transient error?")
requests.prepare({"releases": {}}, "10")
mirror.packages_to_sync = {"Foo": 10}
mirror.record_finished_package = record_finished_package
package = Package("Foo", 10, mirror)
package.sync()
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>\
""")
assert mirror.errors
def test_package_sync_replaces_mismatching_local_files(mirror, requests):
requests.prepare(
{
"releases": {
"0.1": [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
}]
}
},
10,
)
requests.prepare(b"the release content", 10)
os.makedirs("web/packages/any/f/foo")
with open("web/packages/any/f/foo/foo.zip", "wb") as f:
f.write(b"this is not the release content")
mirror.packages_to_sync = {"foo": None}
package = Package("foo", 10, mirror)
package.sync()
assert open("web/packages/any/f/foo/foo.zip").read() == (
"the release content")
def test_package_sync_simple_page_root_uri(mirror, requests):
requests.prepare(
{
"releases": {
"0.1": [
{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"6bd3ddc295176f4dca196b5eb2c4d858",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
},
{
"url":
"https://pypi.example.com/packages/2.7/f/foo/foo.whl",
"filename": "foo.whl",
"digests": {
"md5":
"6bd3ddc295176f4dca196b5eb2c4d858",
"sha256": ("678d78c1ad57455e848081723f7a7a9ff6cdd"
"859b46e9540f574f0a65eb04b0d"),
},
"md5_digest": "6bd3ddc295176f4dca196b5eb2c4d858",
},
]
}
},
10,
)
requests.prepare(b"the release content", 10)
requests.prepare(b"another release content", 10)
mirror.packages_to_sync = {"foo": 10}
mirror.root_uri = "https://files.pythonhosted.org"
package = Package("foo", 10, mirror)
package.sync()
mirror.root_uri = None
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=\
678d78c1ad57455e848081723f7a7a9ff6cdd859b46e9540f574f0a65eb04b0d\
">foo.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/any/f/foo/foo.zip#sha256=\
02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d4e5470a3e4e878bd8cd\
">foo.zip</a><br/>
</body>
</html>\
""")
async def test_sync_incorrect_download_with_old_serials_retries(mirror):
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 2, mirror)
await package.sync()
assert not Path("web/packages/any/f/foo/foo.zip").exists()
assert mirror.errors
def test_package_sync_with_canonical_simple_page(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo": 10}
package = Package("Foo", 10, mirror)
package.sync()
# Cross-check that simple directory hashing is disabled.
assert not os.path.exists("web/simple/f/foo/index.html")
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_save_json_metadata(mirror: Mirror, package_json: Dict[str,
Any]) -> None:
package = Package("foo", 11, mirror)
package.json_file.parent.mkdir(parents=True)
package.json_pypi_symlink.parent.mkdir(parents=True)
package.json_pypi_symlink.symlink_to(Path(gettempdir()))
assert package.save_json_metadata(package_json)
def test_latest_releases_uninitialized(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = {"name": "foo", "version": "2.0.0"}
pkg.releases = {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
pkg._filter_releases()
assert pkg.releases == {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
def test__casing__no__affect(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
Foo<=1.2.0
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.0": {}})
def test__dont__filter__prereleases(self) -> None:
mock_config(
"""\
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
foo<=1.2.0
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {
"1.1.0a2": {},
"1.1.1beta1": {},
"1.2.0": {},
"1.2.1": {},
"1.2.2alpha3": {},
"1.2.3rc1": {},
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test_latest_releases_keep_stable(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
}, # stable version
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {}, # <= stable version, keep it
"2.0.1b1": {},
"2.0.1b2": {}, # <= most recent, keep it
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"2.0.1b2": {}, "2.0.0": {}}
def test_latest_releases_uninitialized(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
def test__filter__matches__package(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(
"""\
[plugins]
enabled =
whitelist_release_pyversion
[whitelist]
python_versions =
foo
"""
)
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = {"name": "foo"}
pkg.releases = {"1.2.0": [{'python_version': 'foo'}, {'python_version': 'foo2'}], "1.2.1": [{'python_version': 'foo2'}]}
pkg._filter_releases()
self.assertListEqual(list(pkg.releases.keys()), ["1.2.0"])
self.assertEqual(len(pkg.releases["1.2.0"]), 1)
self.assertEqual(pkg.releases["1.2.0"][0], {'python_version': 'foo'})
def test__filter__matches__release(self) -> None:
with open(Path(self.tempdir.name) / "requirements.txt", "w") as fh:
fh.write(
"""\
# This is needed for workshop 1
#
foo==1.2.0 # via -r requirements.in
"""
)
mock_config(
f"""\
[plugins]
enabled =
project_requirements
project_requirements_pinned
[allowlist]
requirements_path = {self.tempdir.name}
requirements =
requirements.txt
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {"1.2.0": {}, "1.2.1": {}},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual({"1.2.0": {}}, pkg.releases)
async def test_package_sync_downloads_release_file(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": ""}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert open("web/packages/any/f/foo/foo.zip").read() == ""
def test__filter__matches__release(self) -> None:
mock_config("""\
[plugins]
enabled =
allowlist_release
[whitelist]
packages =
foo==1.2.0
""")
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.0": {}})
async def test_survives_exceptions_from_record_finished_package(
mirror: Mirror) -> None:
def record_finished_package(name: str) -> NoReturn:
import errno
raise OSError(errno.EBADF, "Some transient error?")
mirror.packages_to_sync = {"Foo": 1}
mirror.record_finished_package = record_finished_package # type: ignore
package = Package("Foo", 1, mirror)
await package.sync(mirror.filters)
assert (Path("web/simple/foo/index.html").open().read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
assert mirror.errors
async def test_package_download_rejects_non_package_directory_links(mirror):
mirror.packages_to_sync = {"foo"}
package = Package("foo", 1, mirror)
await package.sync()
assert mirror.errors
assert "foo" in mirror.packages_to_sync
assert not os.path.exists("web/foo/bar/foo/foo.zip")
async def test_package_sync_simple_page_with_existing_dir_with_hash(
mirror_hash_index: BandersnatchMirror,
) -> None:
mirror_hash_index.packages_to_sync = {"foo": 1}
package = Package("foo", serial=1)
os.makedirs(mirror_hash_index.simple_directory(package))
await mirror_hash_index.sync_packages()
assert not os.path.exists("web/simple/foo/index.html")
assert (
open("web/simple/f/foo/index.html").read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
async def test_package_sync_handles_non_pep_503_in_packages_to_sync(master):
with TemporaryDirectory() as td:
mirror = Mirror(td, master, stop_on_error=True)
mirror.packages_to_sync = {"Foo": None}
package = Package("Foo", 1, mirror)
await package.sync()
assert not mirror.errors
def test_package_sync_with_release_no_files_syncs_simple_page_with_hash(
mirror_hash_index, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror_hash_index.packages_to_sync = {"foo": 10}
package = Package("foo", 10, mirror_hash_index)
package.sync()
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
async def test_package_sync_simple_page_root_uri(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": 1}
mirror.root_uri = "https://files.pythonhosted.org"
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
mirror.root_uri = None
expected_root_uri_hrefs = (
'<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=e3b'
+
'0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">foo.whl</a>'
+
'<br/>\n <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.'
+
"zip#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+ '">foo.zip</a><br/>')
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(expected_root_uri_hrefs))
def test_sync_incorrect_download_with_new_serial_fails(mirror, requests):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest":
"b6bcb391b040c4468262706faf9d3cce",
}]
requests.prepare(b"not release content", 11)
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert not os.path.exists(
"web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep))
assert mirror.errors
def test_survives_exceptions_from_record_finished_package(mirror, requests):
def record_finished_package(name):
import errno
raise IOError(errno.EBADF, "Some transient error?")
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo": 10}
mirror.record_finished_package = record_finished_package
package = Package("Foo", 10, mirror)
package.sync()
assert (open("web{0}simple{0}foo{0}index.html".format(sep)).read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
assert mirror.errors
def test_package_sync_does_not_touch_existing_local_file(mirror, requests):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest":
"b6bcb391b040c4468262706faf9d3cce",
}]
requests.prepare(b"the release content", 10)
test_files = [Path("web/packages/any/f/foo/foo.zip")]
touch_files(test_files)
with test_files[0].open("wb") as f:
f.write(b"the release content")
old_stat = test_files[0].stat()
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert old_stat == os.stat(test_files[0])
def test_package_sync_downloads_release_file(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo",
"version": "0.1"
},
"last_serial": 654_321,
"releases": {
"0.1": [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
}]
},
},
10,
)
requests.prepare(b"the release content", 10)
mirror.packages_to_sync = {"foo": None}
package = Package("foo", 10, mirror)
package.sync()
assert open("web/packages/any/f/foo/foo.zip").read() == (
"the release content")
def test_package_sync_simple_page_with_existing_dir_with_hash(
mirror_hash_index, requests):
requests.prepare(
{
"info": {
"name": "foo",
"version": "0.1"
},
"last_serial": 654_321,
"releases": {
"0.1": []
},
},
"10",
)
mirror_hash_index.packages_to_sync = {"foo": 10}
package = Package("foo", 10, mirror_hash_index)
os.makedirs(package.simple_directory)
package.sync()
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_package_sync_with_normalized_simple_page(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo.bar-thing_other": 10}
package = Package("Foo.bar-thing_other", 10, mirror)
package.sync()
# PEP 503 normalization
assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
# Legacy partial normalization as implemented by pip prior to 8.1.2
assert (open("web/simple/foo.bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
# Legacy unnormalized as implemented by pip prior to 6.0
assert (open("web/simple/Foo.bar-thing_other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_gen_data_requires_python(mirror, requests):
fake_no_release = {}
fake_release = {"requires_python": ">=3.6"}
package = Package("foo", 10, mirror)
assert package.gen_data_requires_python(fake_no_release) == ""
assert (package.gen_data_requires_python(fake_release) ==
' data-requires-python=">=3.6"')
