async def test_delete_packages() -> None: args = _fake_args() config = _fake_config() master = Master("https://unittest.org") with TemporaryDirectory() as td: td_path = Path(td) config["mirror"]["directory"] = td web_path = td_path / "web" json_path = web_path / "json" json_path.mkdir(parents=True) pypi_path = web_path / "pypi" pypi_path.mkdir(parents=True) simple_path = web_path / "simple" # Setup web tree with some json, package index.html + fake blobs for package_name in args.pypi_packages: package_simple_path = simple_path / package_name package_simple_path.mkdir(parents=True) package_index_path = package_simple_path / "index.html" package_index_path.touch() package_json_str = MOCK_JSON_TEMPLATE.replace( "PKGNAME", package_name) package_json_path = json_path / package_name with package_json_path.open("w") as pjfp: pjfp.write(package_json_str) legacy_json_path = pypi_path / package_name / "json" legacy_json_path.parent.mkdir() legacy_json_path.symlink_to(package_json_path) package_json = loads(package_json_str) for _version, blobs in package_json["releases"].items(): for blob in blobs: url_parts = urlparse(blob["url"]) blob_path = web_path / url_parts.path[1:] blob_path.parent.mkdir(parents=True, exist_ok=True) blob_path.touch() # See we have a correct mirror setup assert find(web_path) == EXPECTED_WEB_BEFORE_DELETION args.dry_run = True assert await delete_packages(config, args, master) == 0 args.dry_run = False with patch("bandersnatch.delete.logger.info") as mock_log: assert await delete_packages(config, args, master) == 0 assert mock_log.call_count == 1 # See we've deleted it all assert find(web_path) == EXPECTED_WEB_AFTER_DELETION
def test_mirror_empty_master_gets_index(mirror): mirror.master.all_packages = asynctest.asynctest.CoroutineMock(return_value={}) mirror.synchronize() assert """\ last-modified local-stats local-stats{0}days packages simple simple{0}index.html""".format( sep ) == utils.find( mirror.webdir ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> </body> </html>""" ) assert open("status").read() == "0"
def test_mirror_empty_resume_from_todo_list(mirror, requests): response = mock.Mock() response.status_code = 404 requests.prepare(HTTPError(response=response), 10) with open("todo", "w") as todo: todo.write("20\nfoobar 10") mirror.synchronize() assert """\ .lock generation status web web{0}last-modified web{0}local-stats web{0}local-stats/days web{0}packages web{0}simple web{0}simple{0}index.html""".format(sep) == utils.find(mirror.homedir) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> </body> </html>""") assert open("status").read() == "20"
def test_fake_mirror(): expected_mirror_layout = """\ web web/json web/json/bandersnatch web/json/black web/packages web/packages/8f web/packages/8f/1a web/packages/8f/1a/1aa0 web/packages/8f/1a/1aa0/black-2019.6.9.tar.gz web/packages/8f/1a/6969 web/packages/8f/1a/6969/bandersnatch-0.6.9.tar.gz web/packages/8f/1a/6969/black-2018.6.9.tar.gz web/pypi web/pypi/bandersnatch web/pypi/bandersnatch/json web/pypi/black web/pypi/black/json web/simple web/simple/bandersnatch web/simple/bandersnatch/index.html web/simple/black web/simple/black/index.html""" fm = FakeMirror("_mirror_base_test") assert expected_mirror_layout == find(str(fm.mirror_base), True) fm.clean_up()
def test_mirror_sync_package_error_no_early_exit(mirror, requests): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {'foo': 1} requests.prepare( {'releases': { '0.1': [{ 'url': 'https://pypi.example.com/packages/any/f/foo/foo.zip', 'filename': 'foo.zip', 'md5_digest': 'b6bcb391b040c4468262706faf9d3cce'}]}}, 1) requests.prepare(iter('the release content'), 1) mirror.errors = True mirror.synchronize() assert """\ /.lock /generation /todo /web/packages/any/f/foo/foo.zip /web/simple/foo/index.html /web/simple/index.html""" == utils.find(mirror.homedir, dirs=False) assert open('web/simple/index.html').read() == """\ <html><head><title>Simple Index</title></head><body> <a href="foo/">foo</a><br/> </body></html>""" assert open('todo').read() == '1\n'
def test_mirror_sync_package_error_early_exit(mirror, requests): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {'foo': 1} requests.prepare( {'releases': { '0.1': [ {'url': 'https://pypi.example.com/packages/any/f/foo/foo.zip', 'filename': 'foo.zip', 'md5_digest': 'b6bcb391b040c4468262706faf9d3cce'}]}}, 1) requests.prepare(iter('the release content'), 1) with open('web/simple/index.html', 'wb') as index: index.write('old index') mirror.errors = True mirror.stop_on_error = True with pytest.raises(SystemExit): mirror.synchronize() assert """\ /.lock /generation /todo /web/packages/any/f/foo/foo.zip /web/simple/foo/index.html /web/simple/index.html""" == utils.find(mirror.homedir, dirs=False) assert open('web/simple/index.html').read() == 'old index' assert open('todo').read() == '1\n'
def test_mirror_empty_resume_from_todo_list(mirror, requests): response = mock.Mock() response.status_code = 404 requests.prepare(HTTPError(response=response), 10) with open('todo', 'w') as todo: todo.write('20\nfoobar 10') mirror.synchronize() assert """\ /.lock /generation /status /web /web/last-modified /web/local-stats /web/local-stats/days /web/packages /web/simple /web/simple/index.html""" == utils.find(mirror.homedir) assert open('web/simple/index.html').read() == """\ <html><head><title>Simple Index</title></head><body> </body></html>""" assert open('status').read() == '20'
def test_mirror_sync_package_error_no_early_exit(mirror, requests): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {"foo": 1} requests.prepare( { "info": { "name": "foo" }, "last_serial": 654_321, "releases": { "0.1": [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d4" "e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] }, }, 1, ) requests.prepare(b"the release content", 1) mirror.errors = True changed_packages = mirror.synchronize() assert """\ .lock generation todo web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format(sep) == utils.find(mirror.homedir, dirs=False) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") assert open("todo").read() == "1\n" # Check the returned dict is accurate expected = { "foo": {"web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep)} } assert changed_packages == expected
async def mirror_sync_package_error_early_exit(mirror: Mirror) -> None: mirror.master.all_packages = asynctest.CoroutineMock( # type: ignore return_value={"foo": 1} ) with Path("web/simple/index.html").open("wb") as index: index.write(b"old index") mirror.errors = True mirror.stop_on_error = True with pytest.raises(SystemExit): await mirror.synchronize() assert """\ .lock generation todo web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format( sep ) == utils.find( mirror.homedir, dirs=False ) assert open("web{0}simple{0}index.html".format(sep)).read() == "old index" assert open("todo").read() == "1\n"
async def test_mirror_sync_package(mirror: BandersnatchMirror) -> None: mirror.master.all_packages = mock.AsyncMock(return_value={"foo": 1}) # type: ignore mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() await mirror.synchronize() assert """\ json{0}foo last-modified packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format( sep ) == utils.find( mirror.webdir, dirs=False ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("status", "rb").read() == b"1"
async def test_mirror_empty_master_gets_index(mirror: BandersnatchMirror) -> None: mirror.master.all_packages = mock.AsyncMock(return_value={}) # type: ignore await mirror.synchronize() assert """\ last-modified local-stats local-stats{0}days packages simple simple{0}index.html""".format( sep ) == utils.find( mirror.webdir ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> </body> </html>""" ) assert open("status").read() == "0"
async def test_mirror_sync_package_with_hash( mirror_hash_index: BandersnatchMirror, ) -> None: mirror_hash_index.master.all_packages = mock.AsyncMock( # type: ignore return_value={"foo": 1} ) await mirror_hash_index.synchronize() assert """\ last-modified packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip simple{0}f{0}foo{0}index.html simple{0}index.html""".format( sep ) == utils.find( mirror_hash_index.webdir, dirs=False ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("status").read() == "1"
def test_mirror_sync_package(mirror, requests): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {"foo": 1} mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() requests.prepare(FAKE_RELEASE_DATA, 1) requests.prepare(b"the release content", 1) mirror.master.get = mock.Mock() mirror.master.get.return_value = FAKE_RELEASE_DATA mirror.synchronize() assert """\ json{0}foo last-modified packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format(sep) == utils.find(mirror.webdir, dirs=False) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") assert open("status", "rb").read() == b"1"
async def test_sync_specific_packages(mirror: BandersnatchMirror) -> None: FAKE_SERIAL = b"112233" with open("status", "wb") as f: f.write(FAKE_SERIAL) # Package names should be normalized by synchronize() specific_packages = ["Foo"] mirror.master.all_packages = AsyncMock(return_value={"foo": 1}) # type: ignore mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() await mirror.synchronize(specific_packages) assert """\ json{0}foo packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format(sep) == utils.find(mirror.webdir, dirs=False) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") # The "sync" method shouldn't update the serial assert open("status", "rb").read() == FAKE_SERIAL
def test_fake_mirror(): expected_mirror_layout = """\ web web{0}json web{0}json{0}bandersnatch web{0}json{0}black web{0}packages web{0}packages{0}8f web{0}packages{0}8f{0}1a web{0}packages{0}8f{0}1a{0}1aa0 web{0}packages{0}8f{0}1a{0}1aa0{0}black-2019.6.9.tar.gz web{0}packages{0}8f{0}1a{0}6969 web{0}packages{0}8f{0}1a{0}6969{0}bandersnatch-0.6.9.tar.gz web{0}packages{0}8f{0}1a{0}6969{0}black-2018.6.9.tar.gz web{0}pypi web{0}pypi{0}bandersnatch web{0}pypi{0}bandersnatch{0}json web{0}pypi{0}black web{0}pypi{0}black{0}json web{0}simple web{0}simple{0}bandersnatch web{0}simple{0}bandersnatch{0}index.html web{0}simple{0}black web{0}simple{0}black{0}index.html""".format(os.sep) fm = FakeMirror("_mirror_base_test") assert expected_mirror_layout == find(str(fm.mirror_base), True) fm.clean_up()
async def test_mirror_serial_current_no_sync_of_packages_and_index_page( mirror): mirror.master.changed_packages = asynctest.CoroutineMock(return_value={}) mirror.synced_serial = 1 await mirror.synchronize() assert """\ last-modified""" == utils.find(mirror.webdir, dirs=False)
async def test_mirror_serial_current_no_sync_of_packages_and_index_page( mirror: BandersnatchMirror, ) -> None: mirror.master.changed_packages = mock.AsyncMock( return_value={}) # type: ignore mirror.synced_serial = 1 await mirror.synchronize() assert """\ last-modified""" == utils.find(mirror.webdir, dirs=False)
def test_mirror_sync_package_with_hash(mirror_hash_index, requests): mirror_hash_index.master.all_packages = asynctest.CoroutineMock( return_value={"foo": 1} ) requests.prepare( { "info": {"name": "foo", "version": "0.1"}, "last_serial": 654_321, "releases": { "0.1": [ { "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ( "02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d" "4e5470a3e4e878bd8cd" ), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", } ] }, }, 1, ) requests.prepare(b"the release content", 1) mirror_hash_index.synchronize() assert """\ last-modified packages{0}any{0}f{0}foo{0}foo.zip simple{0}f{0}foo{0}index.html simple{0}index.html""".format( sep ) == utils.find( mirror_hash_index.webdir, dirs=False ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("status").read() == "1"
def test_mirror_serial_current_no_sync_of_packages_and_index_page( mirror, requests): mirror.master.changed_packages = mock.Mock() mirror.master.changed_packages.return_value = {} mirror.synced_serial = 1 mirror.synchronize() assert """\ last-modified""" == utils.find(mirror.webdir, dirs=False)
def test_mirror_serial_current_no_sync_of_packages_and_index_page( mirror, requests): mirror.master.changed_packages = mock.Mock() mirror.master.changed_packages.return_value = {} mirror.synced_serial = 1 mirror.synchronize() assert """\ /last-modified""" == utils.find(mirror.webdir, dirs=False)
def test_mirror_sync_package_error_early_exit(mirror, requests): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {"foo": 1} requests.prepare( { "info": { "name": "foo", "version": "0.1" }, "last_serial": 654_321, "releases": { "0.1": [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0" "d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] }, }, 1, ) requests.prepare(b"the release content", 1) with open("web{0}simple{0}index.html".format(sep), "wb") as index: index.write(b"old index") mirror.errors = True mirror.stop_on_error = True with pytest.raises(SystemExit): mirror.synchronize() assert """\ .lock generation todo web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format(sep) == utils.find(mirror.homedir, dirs=False) assert open("web{0}simple{0}index.html".format(sep)).read() == "old index" assert open("todo").read() == "1\n"
async def test_mirror_empty_resume_from_todo_list(mirror: BandersnatchMirror) -> None: with open("todo", "w") as todo: todo.write("20\nfoobar 1") await mirror.synchronize() expected = """\ .lock generation status web web{0}last-modified web{0}local-stats web{0}local-stats{0}days web{0}packages web{0}packages{0}2.7 web{0}packages{0}2.7{0}f web{0}packages{0}2.7{0}f{0}foo web{0}packages{0}2.7{0}f{0}foo{0}foo.whl web{0}packages{0}any web{0}packages{0}any{0}f web{0}packages{0}any{0}f{0}foo web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple web{0}simple{0}foobar web{0}simple{0}foobar{0}index.html web{0}simple{0}index.html""".format( sep ) if WINDOWS: expected = expected.replace(".lock\n", "") assert expected == utils.find(mirror.homedir) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foobar/">foobar</a><br/> </body> </html>""" ) assert open("status").read() == "20"
def test_mirror_empty_master_gets_index(mirror): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {} mirror.synchronize() assert """\ /last-modified /local-stats /local-stats/days /packages /simple /simple/index.html""" == utils.find(mirror.webdir) assert open('web/simple/index.html').read() == """\ <html><head><title>Simple Index</title></head><body> </body></html>""" assert open('status').read() == '0'
def test_mirror_sync_package_with_hash(mirror_hash_index, requests): mirror_hash_index.master.all_packages = mock.Mock() mirror_hash_index.master.all_packages.return_value = {"foo": 1} requests.prepare( { "releases": { "0.1": [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d" "4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] } }, 1, ) requests.prepare(b"the release content", 1) mirror_hash_index.synchronize() assert """\ /last-modified /packages/any/f/foo/foo.zip /simple/f/foo/index.html /simple/index.html""" == utils.find(mirror_hash_index.webdir, dirs=False) assert (open("web/simple/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") assert open("status").read() == "1"
async def test_mirror_sync_package_error_no_early_exit( mirror: BandersnatchMirror, ) -> None: mirror.master.all_packages = mock.AsyncMock(return_value={"foo": 1}) # type: ignore mirror.errors = True changed_packages = await mirror.synchronize() expected = """\ .lock generation todo web{0}packages{0}2.7{0}f{0}foo{0}foo.whl web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format( sep ) if WINDOWS: expected = expected.replace(".lock\n", "") assert expected == utils.find(mirror.homedir, dirs=False) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("todo").read() == "1\n" # Check the returned dict is accurate expected_dict = { "foo": { "web{0}packages{0}2.7{0}f{0}foo{0}foo.whl".format(sep), "web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep), } } assert changed_packages == expected_dict
def test_mirror_empty_master_gets_index(mirror): mirror.master.all_packages = mock.Mock() mirror.master.all_packages.return_value = {} mirror.synchronize() assert """\ /last-modified /local-stats /local-stats/days /packages /simple /simple/index.html""" == utils.find(mirror.webdir) assert (open("web/simple/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> </body> </html>""") assert open("status").read() == "0"
async def test_mirror_sync_package_download_mirror_fallback( mirror: BandersnatchMirror, ) -> None: mirror.master.all_packages = mock.AsyncMock(return_value={"foo": 1}) # type: ignore mirror.json_save = True # Recall bootstrap so we have the json dirs mirror._bootstrap() # This download mirror URL does not work, should fallback to normal logic mirror.download_mirror = "https://not-working.example.com/pypi" await mirror.synchronize() assert """\ json{0}foo last-modified packages{0}2.7{0}f{0}foo{0}foo.whl packages{0}any{0}f{0}foo{0}foo.zip pypi{0}foo{0}json simple{0}foo{0}index.html simple{0}index.html""".format( sep ) == utils.find( mirror.webdir, dirs=False ) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" )
async def test_mirror_sync_package_error_no_early_exit(mirror): mirror.master.all_packages = asynctest.CoroutineMock( return_value={"foo": 1}) mirror.errors = True changed_packages = await mirror.synchronize() assert """\ .lock generation todo web{0}packages{0}2.7{0}f{0}foo{0}foo.whl web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format(sep) == utils.find(mirror.homedir, dirs=False) assert (open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""") assert open("todo").read() == "1\n" # Check the returned dict is accurate expected = { "foo": { "web{0}packages{0}2.7{0}f{0}foo{0}foo.whl".format(sep), "web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep), } } assert changed_packages == expected