def test__filter__varying__specifiers(self) -> None:
mock_config(
"""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
allowlist_project
[allowlist]
packages =
foo==1.2.3
bar~=3.0,<=1.5
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"bar": "",
"snu": "",
}
mirror._filter_packages()
self.assertEqual({"foo": "", "bar": ""}, mirror.packages_to_sync)
async def test_sync_specific_packages(mirror: BandersnatchMirror) -> None:
FAKE_SERIAL = b"112233"
with open("status", "wb") as f:
f.write(FAKE_SERIAL)
# Package names should be normalized by synchronize()
specific_packages = ["Foo"]
mirror.master.all_packages = AsyncMock(return_value={"foo":
1}) # type: ignore
mirror.json_save = True
# Recall bootstrap so we have the json dirs
mirror._bootstrap()
await mirror.synchronize(specific_packages)
assert """\
json{0}foo
packages{0}2.7{0}f{0}foo{0}foo.whl
packages{0}any{0}f{0}foo{0}foo.zip
pypi{0}foo{0}json
simple{0}foo{0}index.html
simple{0}index.html""".format(sep) == utils.find(mirror.webdir, dirs=False)
assert (open("web{0}simple{0}index.html".format(sep)).read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Simple Index</title>
</head>
<body>
<a href="foo/">foo</a><br/>
</body>
</html>""")
# The "sync" method shouldn't update the serial
assert open("status", "rb").read() == FAKE_SERIAL
async def test_package_sync_handles_non_pep_503_in_packages_to_sync(
master: Master, ) -> None:
with TemporaryDirectory() as td:
mirror = BandersnatchMirror(Path(td), master, stop_on_error=True)
mirror.packages_to_sync = {"Foo": 1}
await mirror.sync_packages()
assert not mirror.errors
def test_gen_data_requires_python(mirror: BandersnatchMirror) -> None:
fake_no_release: Dict[str, str] = {}
fake_release = {"requires_python": ">=3.6"}
assert mirror.gen_data_requires_python(fake_no_release) == ""
assert (mirror.gen_data_requires_python(fake_release) ==
' data-requires-python=">=3.6"')
def test_find_package_indexes_in_dir_threaded(
mirror: BandersnatchMirror) -> None:
directories = (
"web/simple/peerme",
"web/simple/click",
"web/simple/zebra",
"web/simple/implicit",
"web/simple/pyaib",
"web/simple/setuptools",
)
with TemporaryDirectory() as td:
# Create local mirror first so we '_bootstrap'
mirror_base = Path(td)
local_mirror = BandersnatchMirror(mirror_base,
mirror.master,
stop_on_error=True)
# Create fake file system objects
for directory in directories:
(mirror_base / directory).mkdir(parents=True, exist_ok=True)
with (mirror_base / "web/simple/index.html").open("w") as index:
index.write("<html></html>")
packages = local_mirror.find_package_indexes_in_dir(mirror_base /
"web/simple")
assert "index.html" not in packages # This should never be in the list
assert len(packages) == 6 # We expect 6 packages with 6 dirs created
assert packages[0] == "click" # Check sorted - click should be first
async def test_package_sync_simple_page_root_uri(
mirror: BandersnatchMirror) -> None:
mirror.packages_to_sync = {"foo": 1}
mirror.root_uri = "https://files.pythonhosted.org"
await mirror.sync_packages()
mirror.root_uri = None
expected_root_uri_hrefs = (
'<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=e3b'
+
'0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">foo.whl</a>'
+
'<br/>\n <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.'
+
"zip#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+ '">foo.zip</a><br/>')
assert (open("web/simple/foo/index.html").read() == """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(expected_root_uri_hrefs))
async def test_package_sync_skips_release_file(mirror: BandersnatchMirror) -> None:
mirror.release_files_save = False
mirror.packages_to_sync = {"foo": 0}
await mirror.sync_packages()
assert not mirror.errors
assert not os.path.exists("web/packages/any/f/foo/foo.zip")
def test_mirror_with_same_homedir_needs_lock(mirror: BandersnatchMirror,
tmpdir: Path) -> None:
try:
BandersnatchMirror(mirror.homedir, mirror.master)
except RuntimeError:
pass
BandersnatchMirror(mirror.homedir / "test", mirror.master)
def test__filter__find_files(self) -> None:
absolute_file_path = Path(self.tempdir.name) / "requirements.txt"
with open(absolute_file_path, "w") as fh:
fh.write("""\
# This is needed for workshop 1
#
foo==1.2.0 # via -r requirements.in
""")
mock_config(f"""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
project_requirements
[allowlist]
requirements =
{absolute_file_path}
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"bar": "",
"baz": "",
}
mirror._filter_packages()
self.assertEqual({"foo": ""}, mirror.packages_to_sync)
async def test_package_sync_simple_page_with_existing_dir_with_hash(
mirror_hash_index: BandersnatchMirror,
) -> None:
mirror_hash_index.packages_to_sync = {"foo": 1}
package = Package("foo", serial=1)
os.makedirs(mirror_hash_index.simple_directory(package))
await mirror_hash_index.sync_packages()
assert not os.path.exists("web/simple/foo/index.html")
assert (
open("web/simple/f/foo/index.html").read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
async def test_mirror_sync_package(mirror: BandersnatchMirror) -> None:
mirror.master.all_packages = mock.AsyncMock(return_value={"foo": 1}) # type: ignore
mirror.json_save = True
# Recall bootstrap so we have the json dirs
mirror._bootstrap()
await mirror.synchronize()
assert """\
json{0}foo
last-modified
packages{0}2.7{0}f{0}foo{0}foo.whl
packages{0}any{0}f{0}foo{0}foo.zip
pypi{0}foo{0}json
simple{0}foo{0}index.html
simple{0}index.html""".format(
sep
) == utils.find(
mirror.webdir, dirs=False
)
assert (
open("web{0}simple{0}index.html".format(sep)).read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Simple Index</title>
</head>
<body>
<a href="foo/">foo</a><br/>
</body>
</html>"""
)
assert open("status", "rb").read() == b"1"
def test__filter__commented__out(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_project
[allowlist]
packages =
foo==1.2.3 # inline comment
# bar
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"bar": "",
"snu": "",
}
mirror._filter_packages()
self.assertEqual({"foo": ""}, mirror.packages_to_sync)
async def test_survives_exceptions_from_record_finished_package(
mirror: BandersnatchMirror,
) -> None:
def record_finished_package(name: str) -> NoReturn:
import errno
raise OSError(errno.EBADF, "Some transient error?")
mirror.packages_to_sync = {"Foo": 1}
mirror.record_finished_package = record_finished_package # type: ignore
await mirror.sync_packages()
assert (
Path("web/simple/foo/index.html").open().read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
assert mirror.errors
def test_gen_html_file_tags(mirror: BandersnatchMirror) -> None:
fake_no_release: Dict[str, str] = {}
# only requires_python
fake_release_1 = {"requires_python": ">=3.6"}
# only data_yanked
fake_release_2 = {"yanked": True, "yanked_reason": "Broken release"}
# requires_python and data_yanked
fake_release_3 = {
"requires_python": ">=3.6",
"yanked": True,
"yanked_reason": "Broken release",
}
assert mirror.gen_html_file_tags(fake_no_release) == ""
assert (
mirror.gen_html_file_tags(fake_release_1) == ' data-requires-python=">=3.6"'
)
assert mirror.gen_html_file_tags(fake_release_2) == ' data-yanked="Broken release"'
assert (
mirror.gen_html_file_tags(fake_release_3)
== ' data-requires-python=">=3.6" data-yanked="Broken release"'
)
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo-good": "",
"foo-evil": "",
"foo-neutral": ""
}
mirror._filter_packages()
assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
def test_mirror_json_metadata(mirror: BandersnatchMirror,
package_json: Dict[str, Any]) -> None:
package = Package("foo", serial=11)
mirror.json_file(package.name).parent.mkdir(parents=True)
mirror.json_pypi_symlink(package.name).parent.mkdir(parents=True)
mirror.json_pypi_symlink(package.name).symlink_to(Path(gettempdir()))
assert mirror.save_json_metadata(package_json, package.name)
assert mirror.json_pypi_symlink(package.name).is_symlink()
assert Path("../../json/foo") == Path(
os.readlink(str(mirror.json_pypi_symlink(package.name))))
def test__filter__nomatch_package(self) -> None:
mock_config("""\
[blocklist]
plugins =
blocklist_project
packages =
foo
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo2": ""}
mirror._filter_packages()
self.assertIn("foo2", mirror.packages_to_sync.keys())
def test_validate_todo(mirror: BandersnatchMirror) -> None:
valid_todo = "69\ncooper 69\ndan 1\n"
invalid_todo = "cooper l33t\ndan n00b\n"
with TemporaryDirectory() as td:
test_mirror = BandersnatchMirror(Path(td), mirror.master)
for todo_data in (valid_todo, invalid_todo):
with test_mirror.todolist.open("w") as tdfp:
tdfp.write(todo_data)
test_mirror._validate_todo()
if todo_data == valid_todo:
assert test_mirror.todolist.exists()
else:
assert not test_mirror.todolist.exists()
def test_mirror_recovers_from_inconsistent_serial(tmpdir: Path) -> None:
with open(str(tmpdir / "generation"), "w") as generation:
generation.write("")
with open(str(tmpdir / "status"), "w") as status:
status.write("1234")
m = BandersnatchMirror(tmpdir, mock.Mock())
assert m.synced_serial == 0
def test_mirror_loads_serial(tmpdir: Path) -> None:
with open(str(tmpdir / "generation"), "w") as generation:
generation.write("5")
with open(str(tmpdir / "status"), "w") as status:
status.write("1234")
m = BandersnatchMirror(tmpdir, mock.Mock())
assert m.synced_serial == 1234
async def test_package_sync_with_normalized_simple_page(
mirror: BandersnatchMirror,
) -> None:
mirror.packages_to_sync = {"Foo.bar-thing_other": 1}
await mirror.sync_packages()
# PEP 503 normalization
assert (
open("web/simple/foo-bar-thing-other/index.html").read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo.bar-thing_other</title>
</head>
<body>
<h1>Links for Foo.bar-thing_other</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
async def test_package_sync_with_canonical_simple_page_with_hash(
mirror_hash_index: BandersnatchMirror,
) -> None:
mirror_hash_index.packages_to_sync = {"Foo": 1}
await mirror_hash_index.sync_packages()
assert not os.path.exists("web/simple/foo/index.html")
assert (
open("web/simple/f/foo/index.html").read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for Foo</title>
</head>
<body>
<h1>Links for Foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
async def test_package_sync_with_release_no_files_syncs_simple_page(
mirror: BandersnatchMirror,
) -> None:
mirror.packages_to_sync = {"foo": 1}
await mirror.sync_packages()
# Cross-check that simple directory hashing is disabled.
assert not os.path.exists("web/simple/f/foo/index.html")
assert (
open("web/simple/foo/index.html").read()
== """\
<!DOCTYPE html>
<html>
<head>
<title>Links for foo</title>
</head>
<body>
<h1>Links for foo</h1>
{}
</body>
</html>
<!--SERIAL 654321-->\
""".format(
EXPECTED_REL_HREFS
)
)
async def test_sync_incorrect_download_with_old_serials_retries(
mirror: BandersnatchMirror, ) -> None:
mirror.packages_to_sync = {"foo": 2}
await mirror.sync_packages()
assert not Path("web/packages/any/f/foo/foo.zip").exists()
assert mirror.errors
async def get_package_from_pypi(package_name, plugin_path):
"""
Download a package from PyPI.
:param name: name of the package to download from PyPI
:return: String path to the package
"""
config = BandersnatchConfig().config
config["mirror"]["master"] = "https://pypi.org"
config["mirror"]["workers"] = "1"
config["mirror"]["directory"] = plugin_path
if not config.has_section("plugins"):
config.add_section("plugins")
config["plugins"]["enabled"] = "blocklist_release\n"
if not config.has_section("allowlist"):
config.add_section("allowlist")
config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n"
config["allowlist"]["packages"] = "\n".join([package_name])
os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True)
async with Master("https://pypi.org/") as master:
mirror = BandersnatchMirror(homedir=plugin_path, master=master)
name = Requirement(package_name).name
result = await mirror.synchronize([name])
package_found = False
for package in result[name]:
current_path = os.path.join(plugin_path, package)
destination_path = os.path.join(plugin_path, "dist",
os.path.basename(package))
shutil.move(current_path, destination_path)
package_found = True
return package_found
async def test_package_download_rejects_non_package_directory_links(
mirror: BandersnatchMirror, ) -> None:
mirror.packages_to_sync = {"foo"} # type: ignore
await mirror.sync_packages()
assert mirror.errors
assert "foo" in mirror.packages_to_sync
assert not os.path.exists("web/foo/bar/foo/foo.zip")
async def test_package_sync_downloads_release_file(
mirror: BandersnatchMirror) -> None:
mirror.packages_to_sync = {"foo": 0}
await mirror.sync_packages()
assert not mirror.errors
assert open("web/packages/any/f/foo/foo.zip").read() == ""
async def test_package_sync_with_error_keeps_it_on_todo_list(
mirror: BandersnatchMirror, ) -> None:
# Make packages_to_sync to generate an error
mirror.packages_to_sync = {"foo"} # type: ignore
await mirror.sync_packages()
assert mirror.errors
assert "foo" in mirror.packages_to_sync
def test__dont__filter__prereleases(self) -> None:
mock_config(
"""\
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
foo<=1.2.0
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {
"1.1.0a2": {},
"1.1.1beta1": {},
"1.2.0": {},
"1.2.1": {},
"1.2.2alpha3": {},
"1.2.3rc1": {},
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test__casing__no__affect(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
Foo<=1.2.0
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.0": {}})
