def test__filter__varying__specifiers(self) -> None: mock_config( """\ [mirror] storage-backend = filesystem [plugins] enabled = allowlist_project [allowlist] packages = foo==1.2.3 bar~=3.0,<=1.5 """ ) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = { "foo": "", "bar": "", "snu": "", } mirror._filter_packages() self.assertEqual({"foo": "", "bar": ""}, mirror.packages_to_sync)
async def test_sync_specific_packages(mirror: BandersnatchMirror) -> None: FAKE_SERIAL = b"112233" with open("status", "wb") as f: f.write(FAKE_SERIAL) # Package names should be normalized by synchronize() specific_packages = ["Foo"] mirror.master.all_packages = AsyncMock(return_value={"foo": 1}) # type: ignore mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() await mirror.synchronize(specific_packages) assert """\ json{0}foo packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format(sep) == utils.find(mirror.webdir, dirs=False) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") # The "sync" method shouldn't update the serial assert open("status", "rb").read() == FAKE_SERIAL
async def test_package_sync_handles_non_pep_503_in_packages_to_sync( master: Master, ) -> None: with TemporaryDirectory() as td: mirror = BandersnatchMirror(Path(td), master, stop_on_error=True) mirror.packages_to_sync = {"Foo": 1} await mirror.sync_packages() assert not mirror.errors
def test_gen_data_requires_python(mirror: BandersnatchMirror) -> None: fake_no_release: Dict[str, str] = {} fake_release = {"requires_python": ">=3.6"} assert mirror.gen_data_requires_python(fake_no_release) == "" assert (mirror.gen_data_requires_python(fake_release) == ' data-requires-python=">=3.6"')
def test_find_package_indexes_in_dir_threaded( mirror: BandersnatchMirror) -> None: directories = ( "web/simple/peerme", "web/simple/click", "web/simple/zebra", "web/simple/implicit", "web/simple/pyaib", "web/simple/setuptools", ) with TemporaryDirectory() as td: # Create local mirror first so we '_bootstrap' mirror_base = Path(td) local_mirror = BandersnatchMirror(mirror_base, mirror.master, stop_on_error=True) # Create fake file system objects for directory in directories: (mirror_base / directory).mkdir(parents=True, exist_ok=True) with (mirror_base / "web/simple/index.html").open("w") as index: index.write("<html></html>") packages = local_mirror.find_package_indexes_in_dir(mirror_base / "web/simple") assert "index.html" not in packages # This should never be in the list assert len(packages) == 6 # We expect 6 packages with 6 dirs created assert packages[0] == "click" # Check sorted - click should be first
async def test_package_sync_simple_page_root_uri( mirror: BandersnatchMirror) -> None: mirror.packages_to_sync = {"foo": 1} mirror.root_uri = "https://files.pythonhosted.org" await mirror.sync_packages() mirror.root_uri = None expected_root_uri_hrefs = ( '<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=e3b' + '0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">foo.whl</a>' + '<br/>\n <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.' + "zip#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + '">foo.zip</a><br/>') assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(expected_root_uri_hrefs))
async def test_package_sync_skips_release_file(mirror: BandersnatchMirror) -> None: mirror.release_files_save = False mirror.packages_to_sync = {"foo": 0} await mirror.sync_packages() assert not mirror.errors assert not os.path.exists("web/packages/any/f/foo/foo.zip")
def test_mirror_with_same_homedir_needs_lock(mirror: BandersnatchMirror, tmpdir: Path) -> None: try: BandersnatchMirror(mirror.homedir, mirror.master) except RuntimeError: pass BandersnatchMirror(mirror.homedir / "test", mirror.master)
def test__filter__find_files(self) -> None: absolute_file_path = Path(self.tempdir.name) / "requirements.txt" with open(absolute_file_path, "w") as fh: fh.write("""\ # This is needed for workshop 1 # foo==1.2.0 # via -r requirements.in """) mock_config(f"""\ [mirror] storage-backend = filesystem workers = 2 [plugins] enabled = project_requirements [allowlist] requirements = {absolute_file_path} """) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = { "foo": "", "bar": "", "baz": "", } mirror._filter_packages() self.assertEqual({"foo": ""}, mirror.packages_to_sync)
async def test_package_sync_simple_page_with_existing_dir_with_hash( mirror_hash_index: BandersnatchMirror, ) -> None: mirror_hash_index.packages_to_sync = {"foo": 1} package = Package("foo", serial=1) os.makedirs(mirror_hash_index.simple_directory(package)) await mirror_hash_index.sync_packages() assert not os.path.exists("web/simple/foo/index.html") assert ( open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format( EXPECTED_REL_HREFS ) )
async def test_mirror_sync_package(mirror: BandersnatchMirror) -> None: mirror.master.all_packages = mock.AsyncMock(return_value={"foo": 1}) # type: ignore mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() await mirror.synchronize() assert """\ json{0}foo last-modified packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format( sep ) == utils.find( mirror.webdir, dirs=False ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("status", "rb").read() == b"1"
def test__filter__commented__out(self) -> None: mock_config("""\ [mirror] storage-backend = filesystem workers = 2 [plugins] enabled = allowlist_project [allowlist] packages = foo==1.2.3 # inline comment # bar """) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = { "foo": "", "bar": "", "snu": "", } mirror._filter_packages() self.assertEqual({"foo": ""}, mirror.packages_to_sync)
async def test_survives_exceptions_from_record_finished_package( mirror: BandersnatchMirror, ) -> None: def record_finished_package(name: str) -> NoReturn: import errno raise OSError(errno.EBADF, "Some transient error?") mirror.packages_to_sync = {"Foo": 1} mirror.record_finished_package = record_finished_package # type: ignore await mirror.sync_packages() assert ( Path("web/simple/foo/index.html").open().read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format( EXPECTED_REL_HREFS ) ) assert mirror.errors
def test_gen_html_file_tags(mirror: BandersnatchMirror) -> None: fake_no_release: Dict[str, str] = {} # only requires_python fake_release_1 = {"requires_python": ">=3.6"} # only data_yanked fake_release_2 = {"yanked": True, "yanked_reason": "Broken release"} # requires_python and data_yanked fake_release_3 = { "requires_python": ">=3.6", "yanked": True, "yanked_reason": "Broken release", } assert mirror.gen_html_file_tags(fake_no_release) == "" assert ( mirror.gen_html_file_tags(fake_release_1) == ' data-requires-python=">=3.6"' ) assert mirror.gen_html_file_tags(fake_release_2) == ' data-yanked="Broken release"' assert ( mirror.gen_html_file_tags(fake_release_3) == ' data-requires-python=">=3.6" data-yanked="Broken release"' )
def test_plugin_check_match(self) -> None: mock_config(self.config_contents) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = { "foo-good": "", "foo-evil": "", "foo-neutral": "" } mirror._filter_packages() assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
def test_mirror_json_metadata(mirror: BandersnatchMirror, package_json: Dict[str, Any]) -> None: package = Package("foo", serial=11) mirror.json_file(package.name).parent.mkdir(parents=True) mirror.json_pypi_symlink(package.name).parent.mkdir(parents=True) mirror.json_pypi_symlink(package.name).symlink_to(Path(gettempdir())) assert mirror.save_json_metadata(package_json, package.name) assert mirror.json_pypi_symlink(package.name).is_symlink() assert Path("../../json/foo") == Path( os.readlink(str(mirror.json_pypi_symlink(package.name))))
def test__filter__nomatch_package(self) -> None: mock_config("""\ [blocklist] plugins = blocklist_project packages = foo """) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo2": ""} mirror._filter_packages() self.assertIn("foo2", mirror.packages_to_sync.keys())
def test_validate_todo(mirror: BandersnatchMirror) -> None: valid_todo = "69\ncooper 69\ndan 1\n" invalid_todo = "cooper l33t\ndan n00b\n" with TemporaryDirectory() as td: test_mirror = BandersnatchMirror(Path(td), mirror.master) for todo_data in (valid_todo, invalid_todo): with test_mirror.todolist.open("w") as tdfp: tdfp.write(todo_data) test_mirror._validate_todo() if todo_data == valid_todo: assert test_mirror.todolist.exists() else: assert not test_mirror.todolist.exists()
def test_mirror_recovers_from_inconsistent_serial(tmpdir: Path) -> None: with open(str(tmpdir / "generation"), "w") as generation: generation.write("") with open(str(tmpdir / "status"), "w") as status: status.write("1234") m = BandersnatchMirror(tmpdir, mock.Mock()) assert m.synced_serial == 0
def test_mirror_loads_serial(tmpdir: Path) -> None: with open(str(tmpdir / "generation"), "w") as generation: generation.write("5") with open(str(tmpdir / "status"), "w") as status: status.write("1234") m = BandersnatchMirror(tmpdir, mock.Mock()) assert m.synced_serial == 1234
async def test_package_sync_with_normalized_simple_page( mirror: BandersnatchMirror, ) -> None: mirror.packages_to_sync = {"Foo.bar-thing_other": 1} await mirror.sync_packages() # PEP 503 normalization assert ( open("web/simple/foo-bar-thing-other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format( EXPECTED_REL_HREFS ) )
async def test_package_sync_with_canonical_simple_page_with_hash( mirror_hash_index: BandersnatchMirror, ) -> None: mirror_hash_index.packages_to_sync = {"Foo": 1} await mirror_hash_index.sync_packages() assert not os.path.exists("web/simple/foo/index.html") assert ( open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format( EXPECTED_REL_HREFS ) )
async def test_package_sync_with_release_no_files_syncs_simple_page( mirror: BandersnatchMirror, ) -> None: mirror.packages_to_sync = {"foo": 1} await mirror.sync_packages() # Cross-check that simple directory hashing is disabled. assert not os.path.exists("web/simple/f/foo/index.html") assert ( open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format( EXPECTED_REL_HREFS ) )
async def test_sync_incorrect_download_with_old_serials_retries( mirror: BandersnatchMirror, ) -> None: mirror.packages_to_sync = {"foo": 2} await mirror.sync_packages() assert not Path("web/packages/any/f/foo/foo.zip").exists() assert mirror.errors
async def get_package_from_pypi(package_name, plugin_path): """ Download a package from PyPI. :param name: name of the package to download from PyPI :return: String path to the package """ config = BandersnatchConfig().config config["mirror"]["master"] = "https://pypi.org" config["mirror"]["workers"] = "1" config["mirror"]["directory"] = plugin_path if not config.has_section("plugins"): config.add_section("plugins") config["plugins"]["enabled"] = "blocklist_release\n" if not config.has_section("allowlist"): config.add_section("allowlist") config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" config["allowlist"]["packages"] = "\n".join([package_name]) os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) async with Master("https://pypi.org/") as master: mirror = BandersnatchMirror(homedir=plugin_path, master=master) name = Requirement(package_name).name result = await mirror.synchronize([name]) package_found = False for package in result[name]: current_path = os.path.join(plugin_path, package) destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) shutil.move(current_path, destination_path) package_found = True return package_found
async def test_package_download_rejects_non_package_directory_links( mirror: BandersnatchMirror, ) -> None: mirror.packages_to_sync = {"foo"} # type: ignore await mirror.sync_packages() assert mirror.errors assert "foo" in mirror.packages_to_sync assert not os.path.exists("web/foo/bar/foo/foo.zip")
async def test_package_sync_downloads_release_file( mirror: BandersnatchMirror) -> None: mirror.packages_to_sync = {"foo": 0} await mirror.sync_packages() assert not mirror.errors assert open("web/packages/any/f/foo/foo.zip").read() == ""
async def test_package_sync_with_error_keeps_it_on_todo_list( mirror: BandersnatchMirror, ) -> None: # Make packages_to_sync to generate an error mirror.packages_to_sync = {"foo"} # type: ignore await mirror.sync_packages() assert mirror.errors assert "foo" in mirror.packages_to_sync
def test__dont__filter__prereleases(self) -> None: mock_config( """\ [plugins] enabled = allowlist_release [allowlist] packages = foo<=1.2.0 """ ) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1) pkg._metadata = { "info": {"name": "foo"}, "releases": { "1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}, "1.2.1": {}, "1.2.2alpha3": {}, "1.2.3rc1": {}, }, } pkg.filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test__casing__no__affect(self) -> None: mock_config("""\ [mirror] storage-backend = filesystem workers = 2 [plugins] enabled = allowlist_release [allowlist] packages = Foo<=1.2.0 """) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1) pkg._metadata = { "info": { "name": "foo" }, "releases": { "1.2.0": {}, "1.2.1": {} }, } pkg.filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.2.0": {}})