async def test_keep_index_versions_stores_different_prior_versions(
mirror: Mirror, ) -> None:
simple_path = Path("web/simple/foo")
versions_path = simple_path / "versions"
mirror.packages_to_sync = {"foo": 1}
mirror.keep_index_versions = 2
with freeze_time("2018-10-27"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
mirror.packages_to_sync = {"foo": 1}
with freeze_time("2018-10-28"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
version_files = sorted(os.listdir(versions_path))
assert len(version_files) == 2
assert version_files[0].startswith("index_1_2018-10-27")
assert version_files[1].startswith("index_1_2018-10-28")
link_path = simple_path / "index.html"
assert os.path.islink(link_path)
assert os.path.basename(os.readlink(str(link_path))) == version_files[1]
async def test_package_sync_downloads_release_file(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": ""}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert open("web/packages/any/f/foo/foo.zip").read() == ""
async def test_package_sync_handles_non_pep_503_in_packages_to_sync(master):
with TemporaryDirectory() as td:
mirror = Mirror(td, master, stop_on_error=True)
mirror.packages_to_sync = {"Foo": None}
package = Package("Foo", 1, mirror)
await package.sync()
assert not mirror.errors
async def test_survives_exceptions_from_record_finished_package(
mirror: Mirror) -> None:
def record_finished_package(name: str) -> NoReturn:
import errno
raise OSError(errno.EBADF, "Some transient error?")
mirror.packages_to_sync = {"Foo": 1}
mirror.record_finished_package = record_finished_package # type: ignore
package = Package("Foo", 1, mirror)
await package.sync(mirror.filters)
assert (Path("web/simple/foo/index.html").open().read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
assert mirror.errors
async def test_package_sync_simple_page_root_uri(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": 1}
mirror.root_uri = "https://files.pythonhosted.org"
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
mirror.root_uri = None
expected_root_uri_hrefs = (
'<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=e3b'
+
'0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">foo.whl</a>'
+
'<br/>\n <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.'
+
"zip#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+ '">foo.zip</a><br/>')
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(expected_root_uri_hrefs))
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo-good": "", "foo-evil": "", "foo-neutral": ""}
mirror._filter_packages()
assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
async def test_package_download_rejects_non_package_directory_links(
mirror: Mirror, ) -> None:
mirror.packages_to_sync = {"foo"} # type: ignore
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert mirror.errors
assert "foo" in mirror.packages_to_sync
assert not os.path.exists("web/foo/bar/foo/foo.zip")
async def test_package_sync_handles_non_pep_503_in_packages_to_sync(
master: Master, ) -> None:
with TemporaryDirectory() as td:
mirror = Mirror(Path(td), master, stop_on_error=True)
mirror.packages_to_sync = {"Foo": ""}
package = Package("Foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
async def test_sync_incorrect_download_with_old_serials_retries(
mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 2, mirror)
await package.sync(mirror.filters)
assert not Path("web/packages/any/f/foo/foo.zip").exists()
assert mirror.errors
async def test_package_sync_with_error_keeps_it_on_todo_list(
mirror: Mirror) -> None:
# Make packages_to_sync to generate an error
mirror.packages_to_sync = {"foo"} # type: ignore
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert mirror.errors
assert "foo" in mirror.packages_to_sync
def test_plugin_check_match(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo-good": {}, "foo-evil": {}, "foo-neutral": {}}
mirror._filter_packages()
assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
async def test_sync_keeps_superfluous_files_on_nondeleting_mirror(
mirror: Mirror, ) -> None:
test_files = [Path("web/packages/2.4/f/foo/foo.zip")]
touch_files(test_files)
mirror.packages_to_sync = {"foo": ""}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert test_files[0].exists()
async def test_package_sync_replaces_mismatching_local_files(
mirror: Mirror) -> None:
test_files = [Path("web/packages/any/f/foo/foo.zip")]
touch_files(test_files)
with test_files[0].open("wb") as f:
f.write(b"this is not the release content")
mirror.packages_to_sync = {"foo": ""}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert test_files[0].open("r").read() == ""
def test__filter__nomatch_package(self) -> None:
mock_config(
"""\
[blacklist]
plugins =
blacklist_project
packages =
foo
"""
)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo2": ""}
mirror._filter_packages()
self.assertIn("foo2", mirror.packages_to_sync.keys())
async def test_keep_index_versions_stores_one_prior_version(
mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": ""}
mirror.keep_index_versions = 1
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
simple_path = Path("web/simple/foo")
versions_path = simple_path / "versions"
version_files = os.listdir(versions_path)
assert len(version_files) == 1
assert version_files[
0] == f"index_{package.serial}_{make_time_stamp()}.html"
link_path = simple_path / "index.html"
assert link_path.is_symlink()
assert os.path.basename(os.readlink(str(link_path))) == version_files[0]
async def test_package_sync_does_not_touch_existing_local_file(
mirror: Mirror) -> None:
pkg_file_path_str = "web/packages/any/f/foo/foo.zip"
pkg_file_path = Path(pkg_file_path_str)
touch_files((pkg_file_path, ))
with pkg_file_path.open("w") as f:
f.write("")
old_stat = pkg_file_path.stat()
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
# Use Pathlib + create a new object to ensure no caching
# Only compare the relevant stat fields
assert old_stat.st_mtime == Path(pkg_file_path_str).stat().st_mtime
assert old_stat.st_ctime == Path(pkg_file_path_str).stat().st_ctime
def test__filter__nomatch_package(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins =
blacklist_project
packages =
foo
""")
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo2": {}}
mirror._filter_packages()
self.assertIn("foo2", mirror.packages_to_sync.keys())
def test__filter__matches__package(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
whitelist_project
[whitelist]
packages =
foo
""")
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": ""}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
async def test_package_sync_simple_page_with_files(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page_with_hash(
mirror_hash_index: Mirror, ) -> None:
mirror_hash_index.packages_to_sync = {"Foo": 1}
package = Package("Foo", 1, mirror_hash_index)
await package.sync(mirror_hash_index.filters)
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
def test_mirror_filter_packages_match(tmpdir):
"""
Packages that exist in the blacklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[blacklist]
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
for plugin in filter_project_plugins():
plugin.initialize_plugin()
m = Mirror(str(tmpdir), mock.Mock())
m.packages_to_sync = {"example1": None, "example2": None}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir):
"""
Package lines with a PEP440 spec on them should not be filtered from the
list of packages.
"""
test_configuration = """\
[blacklist]
packages =
example3>2.0.0
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
for plugin in filter_project_plugins():
plugin.initialize_plugin()
m = Mirror(str(tmpdir), mock.Mock())
m.packages_to_sync = {"example1": None, "example3": None}
m._filter_packages()
assert "example3" in m.packages_to_sync.keys()
async def test_package_sync_with_normalized_simple_page(
mirror: Mirror) -> None:
mirror.packages_to_sync = {"Foo.bar-thing_other": 1}
package = Package("Foo.bar-thing_other", 1, mirror)
await package.sync(mirror.filters)
# PEP 503 normalization
assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
def test__filter__matches__package(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled =
whitelist_project
[whitelist]
packages =
foo
""")
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
async def test_package_sync_with_canonical_simple_page(mirror: Mirror) -> None:
mirror.packages_to_sync = {"Foo": 1}
package = Package("Foo", 1, mirror)
await package.sync(mirror.filters)
# Cross-check that simple directory hashing is disabled.
assert not os.path.exists("web/simple/f/foo/index.html")
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
def test_mirror_filter_packages_match(tmpdir: Path) -> None:
"""
Packages that exist in the blacklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[plugins]
enabled =
blacklist_project
[blacklist]
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = Mirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example2": ""}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None:
"""
Package lines with a PEP440 spec on them should not be filtered from the
list of packages.
"""
test_configuration = """\
[plugins]
enable =
blacklist_project
[blacklist]
packages =
example3>2.0.0
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = Mirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example3": ""}
m._filter_packages()
assert "example3" in m.packages_to_sync.keys()
def test__filter__nomatch_package(self):
_mock_config(
"""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
whitelist_project
[whitelist]
packages =
foo
"""
)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}, "foo2": {}}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
self.assertNotIn("foo2", mirror.packages_to_sync.keys())
