def test_package_sync_does_not_touch_existing_local_file(mirror, requests):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest":
"b6bcb391b040c4468262706faf9d3cce",
}]
requests.prepare(b"the release content", 10)
test_files = [Path("web/packages/any/f/foo/foo.zip")]
touch_files(test_files)
with test_files[0].open("wb") as f:
f.write(b"the release content")
old_stat = test_files[0].stat()
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert old_stat == os.stat(test_files[0])
def test_package_sync_with_release_no_files_syncs_simple_page_with_hash(
mirror_hash_index, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror_hash_index.packages_to_sync = {"foo": 10}
package = Package("foo", 10, mirror_hash_index)
package.sync()
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_package_sync_simple_page_with_existing_dir_with_hash(
mirror_hash_index, requests):
requests.prepare(
{
"info": {
"name": "foo",
"version": "0.1"
},
"last_serial": 654_321,
"releases": {
"0.1": []
},
},
"10",
)
mirror_hash_index.packages_to_sync = {"foo": 10}
package = Package("foo", 10, mirror_hash_index)
os.makedirs(package.simple_directory)
package.sync()
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_survives_exceptions_from_record_finished_package(mirror, requests):
def record_finished_package(name):
import errno
raise IOError(errno.EBADF, "Some transient error?")
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo": 10}
mirror.record_finished_package = record_finished_package
package = Package("Foo", 10, mirror)
package.sync()
assert (open("web{0}simple{0}foo{0}index.html".format(sep)).read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
assert mirror.errors
def test_package_sync_simple_page_root_uri(mirror, requests):
requests.prepare(
{
"releases": {
"0.1": [
{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"6bd3ddc295176f4dca196b5eb2c4d858",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
},
{
"url":
"https://pypi.example.com/packages/2.7/f/foo/foo.whl",
"filename": "foo.whl",
"digests": {
"md5":
"6bd3ddc295176f4dca196b5eb2c4d858",
"sha256": ("678d78c1ad57455e848081723f7a7a9ff6cdd"
"859b46e9540f574f0a65eb04b0d"),
},
"md5_digest": "6bd3ddc295176f4dca196b5eb2c4d858",
},
]
}
},
10,
)
requests.prepare(b"the release content", 10)
requests.prepare(b"another release content", 10)
mirror.packages_to_sync = {"foo": 10}
mirror.root_uri = "https://files.pythonhosted.org"
package = Package("foo", 10, mirror)
package.sync()
mirror.root_uri = None
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=\
678d78c1ad57455e848081723f7a7a9ff6cdd859b46e9540f574f0a65eb04b0d\
">foo.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/any/f/foo/foo.zip#sha256=\
02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d4e5470a3e4e878bd8cd\
">foo.zip</a><br/>
</body>
</html>\
""")
def test__filter__matches__release(self) -> None:
mock_config("""\
[plugins]
enabled =
allowlist_release
[whitelist]
packages =
foo==1.2.0
""")
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.0": {}})
def test_sync_incorrect_download_with_new_serial_fails(mirror, requests):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest":
"b6bcb391b040c4468262706faf9d3cce",
}]
requests.prepare(b"not release content", 11)
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert not os.path.exists(
"web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep))
assert mirror.errors
def test__casing__no__affect(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
Foo<=1.2.0
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.0": {}})
def test__dont__filter__prereleases(self) -> None:
mock_config(
"""\
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
foo<=1.2.0
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {
"1.1.0a2": {},
"1.1.1beta1": {},
"1.2.0": {},
"1.2.1": {},
"1.2.2alpha3": {},
"1.2.3rc1": {},
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test_latest_releases_keep_stable(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
}, # stable version
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {}, # <= stable version, keep it
"2.0.1b1": {},
"2.0.1b2": {}, # <= most recent, keep it
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"2.0.1b2": {}, "2.0.0": {}}
def test_package_sync_downloads_release_file(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo",
"version": "0.1"
},
"last_serial": 654_321,
"releases": {
"0.1": [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
}]
},
},
10,
)
requests.prepare(b"the release content", 10)
mirror.packages_to_sync = {"foo": None}
package = Package("foo", 10, mirror)
package.sync()
assert open("web/packages/any/f/foo/foo.zip").read() == (
"the release content")
def test_package_sync_replaces_mismatching_local_files(mirror, requests):
requests.prepare(
{
"releases": {
"0.1": [{
"url":
"https://pypi.example.com/packages/any/f/foo/foo.zip",
"filename": "foo.zip",
"digests": {
"md5":
"b6bcb391b040c4468262706faf9d3cce",
"sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c"
"9e87fb0d4e5470a3e4e878bd8cd"),
},
"md5_digest": "b6bcb391b040c4468262706faf9d3cce",
}]
}
},
10,
)
requests.prepare(b"the release content", 10)
os.makedirs("web/packages/any/f/foo")
with open("web/packages/any/f/foo/foo.zip", "wb") as f:
f.write(b"this is not the release content")
mirror.packages_to_sync = {"foo": None}
package = Package("foo", 10, mirror)
package.sync()
assert open("web/packages/any/f/foo/foo.zip").read() == (
"the release content")
def test_survives_exceptions_from_record_finished_package(mirror, requests):
def record_finished_package(name):
import errno
raise IOError(errno.EBADF, "Some transient error?")
requests.prepare({"releases": {}}, "10")
mirror.packages_to_sync = {"Foo": 10}
mirror.record_finished_package = record_finished_package
package = Package("Foo", 10, mirror)
package.sync()
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>\
""")
assert mirror.errors
async def test_keep_index_versions_stores_different_prior_versions(
mirror: Mirror, ) -> None:
simple_path = Path("web/simple/foo")
versions_path = simple_path / "versions"
mirror.packages_to_sync = {"foo": 1}
mirror.keep_index_versions = 2
with freeze_time("2018-10-27"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
mirror.packages_to_sync = {"foo": 1}
with freeze_time("2018-10-28"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
version_files = sorted(os.listdir(versions_path))
assert len(version_files) == 2
assert version_files[0].startswith("index_1_2018-10-27")
assert version_files[1].startswith("index_1_2018-10-28")
link_path = simple_path / "index.html"
assert os.path.islink(link_path)
assert os.path.basename(os.readlink(str(link_path))) == version_files[1]
def test_package_sync_with_canonical_simple_page(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo": 10}
package = Package("Foo", 10, mirror)
package.sync()
# Cross-check that simple directory hashing is disabled.
assert not os.path.exists("web/simple/f/foo/index.html")
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_save_json_metadata(mirror: Mirror, package_json: Dict[str,
Any]) -> None:
package = Package("foo", 11, mirror)
package.json_file.parent.mkdir(parents=True)
package.json_pypi_symlink.parent.mkdir(parents=True)
package.json_pypi_symlink.symlink_to(Path(gettempdir()))
assert package.save_json_metadata(package_json)
def test__filter__matches__release(self) -> None:
with open(Path(self.tempdir.name) / "requirements.txt", "w") as fh:
fh.write(
"""\
# This is needed for workshop 1
#
foo==1.2.0 # via -r requirements.in
"""
)
mock_config(
f"""\
[plugins]
enabled =
project_requirements
project_requirements_pinned
[allowlist]
requirements_path = {self.tempdir.name}
requirements =
requirements.txt
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {"1.2.0": {}, "1.2.1": {}},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual({"1.2.0": {}}, pkg.releases)
def test_latest_releases_uninitialized(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
def test_package_sync_with_normalized_simple_page(mirror, requests):
requests.prepare(
{
"info": {
"name": "foo"
},
"last_serial": 654_321,
"releases": {}
}, "10")
mirror.packages_to_sync = {"Foo.bar-thing_other": 10}
package = Package("Foo.bar-thing_other", 10, mirror)
package.sync()
# PEP 503 normalization
assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
# Legacy partial normalization as implemented by pip prior to 8.1.2
assert (open("web/simple/foo.bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
# Legacy unnormalized as implemented by pip prior to 6.0
assert (open("web/simple/Foo.bar-thing_other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
</body>
</html>
<!--SERIAL 654321-->\
""")
def test_gen_data_requires_python(mirror: Mirror) -> None:
fake_no_release: Dict[str, str] = {}
fake_release = {"requires_python": ">=3.6"}
package = Package("foo", 10, mirror)
assert package.gen_data_requires_python(fake_no_release) == ""
assert (package.gen_data_requires_python(fake_release) ==
' data-requires-python=">=3.6"')
def test_gen_data_requires_python(mirror, requests):
fake_no_release = {}
fake_release = {"requires_python": ">=3.6"}
package = Package("foo", 10, mirror)
assert package.gen_data_requires_python(fake_no_release) == ""
assert (package.gen_data_requires_python(fake_release) ==
' data-requires-python=">=3.6"')
def test_package_sync_with_error_keeps_it_on_todo_list(mirror, requests):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = []
requests.side_effect = Exception
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert "foo" in mirror.packages_to_sync
def setUp_mirror(self) -> None:
self.mirror = Mirror(self.mirror_path,
Master(url="https://foo.bar.com"))
pkg = Package("foobar", 1, self.mirror)
pkg._metadata = {
"info": {
"name": "foobar",
"version": "1.0"
},
"releases": mock.Mock(),
}
self.pkgs.append(pkg)
def test_plugin_check_match(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}
pkg._filter_releases()
assert pkg.releases == {"foo-1.2.0": {}}
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {"name": "foo", "version": "foo-1.2.0"},
"releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"foo-1.2.0": {}}
async def test_cleanup_non_pep_503_paths(mirror):
raw_package_name = "CatDogPython69"
package = Package(raw_package_name, 11, mirror)
await package.cleanup_non_pep_503_paths()
# Create a non normailized directory
touch_files(
[package.mirror.webdir / "simple" / raw_package_name / "index.html"])
package.cleanup = True
with mock.patch("bandersnatch.package.rmtree") as mocked_rmtree:
await package.cleanup_non_pep_503_paths()
assert mocked_rmtree.call_count == 1
def setUp_mirror(self) -> None:
self.master = Master(url="https://foo.bar.com")
self.mirror = BandersnatchMirror(self.mirror_path, self.master,
self.backend)
pkg = Package("foobar", serial=1)
pkg._metadata = {
"info": {
"name": "foobar",
"version": "1.0"
},
"releases": mock.Mock(),
}
self.pkgs.append(pkg)
async def test_package_sync_gives_up_after_3_stale_responses(caplog, mirror):
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = []
pkg_name = "foo"
package = Package(pkg_name, 11, mirror)
package.sleep_on_stale = 0
await package.sync()
assert package.tries == 3
assert mirror.errors
assert "not updating. Giving up" in caplog.text
def test_sync_keeps_superfluous_files_on_nondeleting_mirror(mirror, requests):
test_files = [Path("web/packages/2.4/f/foo/foo.zip")]
touch_files(test_files)
mirror.master.package_releases = mock.Mock()
mirror.master.package_releases.return_value = ["0.1"]
mirror.master.release_urls = mock.Mock()
mirror.master.release_urls.return_value = []
mirror.packages_to_sync = {"foo"}
package = Package("foo", 10, mirror)
package.sync()
assert test_files[0].exists()
def test_latest_releases_ensure_reusable(self) -> None:
"""
Tests the filter multiple times to ensure no state is preserved and
thus is reusable between packages
"""
mock_config(self.config_contents)
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg1 = Package("foo", 1)
pkg1._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"1.0.0": {},
"1.1.0": {},
"1.2.0": {},
"2.0.0": {},
},
}
pkg2 = Package("bar", 1)
pkg2._metadata = {
"info": {
"name": "bar",
"version": "0.3.0"
},
"releases": {
"0.1.0": {},
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"0.1.4": {},
"0.1.5": {},
"0.2.0": {},
"0.3.0": {},
},
}
pkg1.filter_all_releases(mirror.filters.filter_release_plugins())
pkg2.filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg1.releases == {"1.2.0": {}, "2.0.0": {}}
assert pkg2.releases == {"0.2.0": {}, "0.3.0": {}}