def setUp_mirror(self) -> None:
self.mirror = Mirror(self.mirror_path,
Master(url="https://foo.bar.com"))
pkg = Package("foobar", 1, self.mirror)
pkg._metadata = {
"info": {
"name": "foobar",
"version": "1.0"
},
"releases": mock.Mock(),
}
self.pkgs.append(pkg)
def test_mirror_filter_packages_match(tmpdir):
"""
Packages that exist in the blacklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[blacklist]
plugins = blacklist_project
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
for plugin in filter_project_plugins():
plugin.initialize_plugin()
m = Mirror(str(tmpdir), mock.Mock())
m.packages_to_sync = {"example1": None, "example2": None}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test__filter__matches__package(self):
_mock_config(
"""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
whitelist_project
[whitelist]
packages =
foo
"""
)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None:
"""
Package lines with a PEP440 spec on them should not be filtered from the
list of packages.
"""
test_configuration = """\
[plugins]
enable =
blacklist_project
[blacklist]
packages =
example3>2.0.0
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = Mirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example3": ""}
m._filter_packages()
assert "example3" in m.packages_to_sync.keys()
async def test_cleanup_non_pep_503_paths(mirror: Mirror) -> None:
raw_package_name = "CatDogPython69"
package = Package(raw_package_name, 11, mirror)
await mirror.cleanup_non_pep_503_paths(package)
# Create a non normailized directory
touch_files([mirror.webdir / "simple" / raw_package_name / "index.html"])
mirror.cleanup = True
with mock.patch("bandersnatch.mirror.rmtree") as mocked_rmtree:
await mirror.cleanup_non_pep_503_paths(package)
assert mocked_rmtree.call_count == 1
def test_mirror_filter_packages_match(tmpdir: Path) -> None:
"""
Packages that exist in the blacklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[plugins]
enabled =
blacklist_project
[blacklist]
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = Mirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example2": ""}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test_plugin_check_match(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}
pkg._filter_releases()
assert pkg.releases == {"foo-1.2.0": {}}
def test__filter__matches__package(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(
"""\
[plugins]
enabled =
blacklist_project
[blacklist]
packages =
foo
"""
)
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}}
mirror._filter_packages()
self.assertNotIn("foo", mirror.packages_to_sync.keys())
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {"name": "foo", "version": "foo-1.2.0"},
"releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"foo-1.2.0": {}}
async def test_mirror_serial_current_no_sync_of_packages_and_index_page(
mirror: Mirror,
) -> None:
mirror.master.changed_packages = asynctest.CoroutineMock( # type: ignore
return_value={}
)
mirror.synced_serial = 1
await mirror.synchronize()
assert """\
last-modified""" == utils.find(
mirror.webdir, dirs=False
)
def test_mirror_generation_4_resets_status_files(tmpdir):
with open(str(tmpdir / "generation"), "w") as generation:
generation.write("4")
with open(str(tmpdir / "status"), "w") as status:
status.write("1234")
with open(str(tmpdir / "todo"), "w") as status:
status.write("asdf")
m = Mirror(str(tmpdir), mock.Mock())
assert m.synced_serial == 0
assert not os.path.exists(str(tmpdir / "todo"))
assert not os.path.exists(str(tmpdir / "status"))
assert open(str(tmpdir / "generation"), "r").read() == "5"
async def test_package_sync_replaces_mismatching_local_files(
mirror: Mirror) -> None:
test_files = [Path("web/packages/any/f/foo/foo.zip")]
touch_files(test_files)
with test_files[0].open("wb") as f:
f.write(b"this is not the release content")
mirror.packages_to_sync = {"foo": ""}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert test_files[0].open("r").read() == ""
def test__filter__nomatch_package(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(
"""\
[blacklist]
plugins =
whitelist_project
[whitelist]
packages =
foo
"""
)
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}, "foo2": {}}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
self.assertNotIn("foo2", mirror.packages_to_sync.keys())
def test_find_package_indexes_in_dir_threaded(mirror: Mirror) -> None:
directories = (
"web/simple/peerme",
"web/simple/click",
"web/simple/zebra",
"web/simple/implicit",
"web/simple/pyaib",
"web/simple/setuptools",
)
with TemporaryDirectory() as td:
# Create local mirror first so we '_bootstrap'
mirror_base = Path(td)
local_mirror = Mirror(mirror_base, mirror.master, stop_on_error=True)
# Create fake file system objects
for directory in directories:
(mirror_base / directory).mkdir(parents=True, exist_ok=True)
with (mirror_base / "web/simple/index.html").open("w") as index:
index.write("<html></html>")
packages = local_mirror.find_package_indexes_in_dir(mirror_base / "web/simple")
assert "index.html" not in packages # This should never be in the list
assert len(packages) == 6 # We expect 6 packages with 6 dirs created
assert packages[0] == "click" # Check sorted - click should be first
def test__filter__pyversions(self):
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
with open(TEST_JSON) as json_file:
package_json = json.load(json_file)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = package_json["info"]
pkg.releases = package_json["releases"]
pkg._filter_releases()
self.assertEqual(set([v['python_version'] for v in pkg.releases["0.4.3"]]), set(['cp35', 'cp36']))
async def test_package_sync_does_not_touch_existing_local_file(
mirror: Mirror) -> None:
pkg_file_path_str = "web/packages/any/f/foo/foo.zip"
pkg_file_path = Path(pkg_file_path_str)
touch_files((pkg_file_path, ))
with pkg_file_path.open("w") as f:
f.write("")
old_stat = pkg_file_path.stat()
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
# Use Pathlib + create a new object to ensure no caching
# Only compare the relevant stat fields
assert old_stat.st_mtime == Path(pkg_file_path_str).stat().st_mtime
assert old_stat.st_ctime == Path(pkg_file_path_str).stat().st_ctime
async def test_mirror_sync_package_error_no_early_exit(mirror: Mirror) -> None:
mirror.master.all_packages = asynctest.CoroutineMock( # type: ignore
return_value={"foo": 1}
)
mirror.errors = True
changed_packages = await mirror.synchronize()
expected = """\
.lock
generation
todo
web{0}packages{0}2.7{0}f{0}foo{0}foo.whl
web{0}packages{0}any{0}f{0}foo{0}foo.zip
web{0}simple{0}foo{0}index.html
web{0}simple{0}index.html""".format(
sep
)
if WINDOWS:
expected = expected.replace(".lock\n", "")
assert expected == utils.find(mirror.homedir, dirs=False)
assert (
open("web{0}simple{0}index.html".format(sep)).read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Simple Index</title>
</head>
<body>
<a href="foo/">foo</a><br/>
</body>
</html>"""
)
assert open("todo").read() == "1\n"
# Check the returned dict is accurate
expected_dict = {
"foo": {
"web{0}packages{0}2.7{0}f{0}foo{0}foo.whl".format(sep),
"web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep),
}
}
assert changed_packages == expected_dict
async def test_package_sync_simple_page_with_files(mirror: Mirror) -> None:
mirror.packages_to_sync = {"foo": 1}
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
assert not mirror.errors
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
def test_plugin_check_match(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = {"name": "foo", "version": "1.2.0"}
pkg.releases = {
"1.2.0alpha1": {},
"1.2.0a2": {},
"1.2.0beta1": {},
"1.2.0b2": {},
"1.2.0rc1": {},
"1.2.0": {},
}
pkg._filter_releases()
assert pkg.releases == {"1.2.0": {}}
async def test_package_sync_with_normalized_simple_page(
mirror: Mirror) -> None:
mirror.packages_to_sync = {"Foo.bar-thing_other": 1}
package = Package("Foo.bar-thing_other", 1, mirror)
await package.sync(mirror.filters)
# PEP 503 normalization
assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page_with_hash(
mirror_hash_index: Mirror, ) -> None:
mirror_hash_index.packages_to_sync = {"Foo": 1}
package = Package("Foo", 1, mirror_hash_index)
await package.sync(mirror_hash_index.filters)
assert not os.path.exists("web/simple/foo/index.html")
assert (open("web/simple/f/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page(mirror: Mirror) -> None:
mirror.packages_to_sync = {"Foo": 1}
package = Package("Foo", 1, mirror)
await package.sync(mirror.filters)
# Cross-check that simple directory hashing is disabled.
assert not os.path.exists("web/simple/f/foo/index.html")
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(EXPECTED_REL_HREFS))
def test_latest_releases_keep_latest(self):
_mock_config(self.config_contents)
bandersnatch.filter.filter_release_plugins()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = {"name": "foo", "version": "2.0.0"}
pkg.releases = {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
pkg._filter_releases()
assert pkg.releases == {"1.1.3": {}, "2.0.0": {}}
def test__filter__matches__release(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins =
blacklist_release
packages =
foo==1.2.0
""")
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.releases = {"1.2.0": {}, "1.2.1": {}}
pkg._filter_releases()
self.assertEqual(pkg.releases, {"1.2.1": {}})
async def test_keep_index_versions_removes_old_versions(
mirror: Mirror) -> None:
simple_path = Path("web/simple/foo/")
versions_path = simple_path / "versions"
versions_path.mkdir(parents=True)
(versions_path / "index_1_2018-10-26T000000Z.html").touch()
(versions_path / "index_1_2018-10-27T000000Z.html").touch()
mirror.keep_index_versions = 2
with freeze_time("2018-10-28"):
package = Package("foo", 1, mirror)
await package.sync(mirror.filters)
version_files = sorted(f for f in versions_path.iterdir())
assert len(version_files) == 2
assert version_files[0].name.startswith("index_1_2018-10-27")
assert version_files[1].name.startswith("index_1_2018-10-28")
link_path = simple_path / "index.html"
assert link_path.is_symlink()
assert os.path.basename(os.readlink(
str(link_path))) == version_files[1].name
def test__filter__matches__release(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(
"""\
[blacklist]
plugins =
safety_db_release
"""
)
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
self._dump_config(instance)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("aiohttp", 1, mirror)
pkg.info = {"name": "aiohttp"}
pkg.releases = {"0.16.3": {}, "0.16.0": {}, "0.15.1": {}}
pkg._filter_releases()
self.assertEqual(pkg.releases, {"0.16.3": {}})
def test__filter__matches__release(self) -> None:
mock_config(
"""\
[plugins]
enabled =
blacklist_release
[blacklist]
packages =
foo==1.2.0
"""
)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {"1.2.0": {}, "1.2.1": {}},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.1": {}})
def test_latest_releases_keep_latest(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"1.1.3": {}, "2.0.0": {}}
def mirror_hash_index(tmpdir, master, monkeypatch):
monkeypatch.chdir(tmpdir)
from bandersnatch.mirror import Mirror
return Mirror(str(tmpdir), master, hash_index=True)
def mirror(tmpdir, master, monkeypatch):
monkeypatch.chdir(tmpdir)
from bandersnatch.mirror import Mirror
return Mirror(str(tmpdir), master)
