def test_package_sync_does_not_touch_existing_local_file(mirror, requests): mirror.master.package_releases = mock.Mock() mirror.master.package_releases.return_value = ["0.1"] mirror.master.release_urls = mock.Mock() mirror.master.release_urls.return_value = [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c" "9e87fb0d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] requests.prepare(b"the release content", 10) test_files = [Path("web/packages/any/f/foo/foo.zip")] touch_files(test_files) with test_files[0].open("wb") as f: f.write(b"the release content") old_stat = test_files[0].stat() mirror.packages_to_sync = {"foo"} package = Package("foo", 10, mirror) package.sync() assert old_stat == os.stat(test_files[0])
def test_package_sync_with_release_no_files_syncs_simple_page_with_hash( mirror_hash_index, requests): requests.prepare( { "info": { "name": "foo" }, "last_serial": 654_321, "releases": {} }, "10") mirror_hash_index.packages_to_sync = {"foo": 10} package = Package("foo", 10, mirror_hash_index) package.sync() assert not os.path.exists("web/simple/foo/index.html") assert (open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> </body> </html> <!--SERIAL 654321-->\ """)
def test_package_sync_simple_page_with_existing_dir_with_hash( mirror_hash_index, requests): requests.prepare( { "info": { "name": "foo", "version": "0.1" }, "last_serial": 654_321, "releases": { "0.1": [] }, }, "10", ) mirror_hash_index.packages_to_sync = {"foo": 10} package = Package("foo", 10, mirror_hash_index) os.makedirs(package.simple_directory) package.sync() assert not os.path.exists("web/simple/foo/index.html") assert (open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> </body> </html> <!--SERIAL 654321-->\ """)
def test_survives_exceptions_from_record_finished_package(mirror, requests): def record_finished_package(name): import errno raise IOError(errno.EBADF, "Some transient error?") requests.prepare( { "info": { "name": "foo" }, "last_serial": 654_321, "releases": {} }, "10") mirror.packages_to_sync = {"Foo": 10} mirror.record_finished_package = record_finished_package package = Package("Foo", 10, mirror) package.sync() assert (open("web{0}simple{0}foo{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> </body> </html> <!--SERIAL 654321-->\ """) assert mirror.errors
def test_package_sync_simple_page_root_uri(mirror, requests): requests.prepare( { "releases": { "0.1": [ { "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "6bd3ddc295176f4dca196b5eb2c4d858", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c" "9e87fb0d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }, { "url": "https://pypi.example.com/packages/2.7/f/foo/foo.whl", "filename": "foo.whl", "digests": { "md5": "6bd3ddc295176f4dca196b5eb2c4d858", "sha256": ("678d78c1ad57455e848081723f7a7a9ff6cdd" "859b46e9540f574f0a65eb04b0d"), }, "md5_digest": "6bd3ddc295176f4dca196b5eb2c4d858", }, ] } }, 10, ) requests.prepare(b"the release content", 10) requests.prepare(b"another release content", 10) mirror.packages_to_sync = {"foo": 10} mirror.root_uri = "https://files.pythonhosted.org" package = Package("foo", 10, mirror) package.sync() mirror.root_uri = None assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> <a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=\ 678d78c1ad57455e848081723f7a7a9ff6cdd859b46e9540f574f0a65eb04b0d\ ">foo.whl</a><br/> <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.zip#sha256=\ 02db45ea4e09715fbb1ed0fef30d7324db07c9e87fb0d4e5470a3e4e878bd8cd\ ">foo.zip</a><br/> </body> </html>\ """)
def test__filter__matches__release(self) -> None: mock_config("""\ [plugins] enabled = allowlist_release [whitelist] packages = foo==1.2.0 """) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": { "name": "foo" }, "releases": { "1.2.0": {}, "1.2.1": {} }, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.2.0": {}})
def test_sync_incorrect_download_with_new_serial_fails(mirror, requests): mirror.master.package_releases = mock.Mock() mirror.master.package_releases.return_value = ["0.1"] mirror.master.release_urls = mock.Mock() mirror.master.release_urls.return_value = [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c" "9e87fb0d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] requests.prepare(b"not release content", 11) mirror.packages_to_sync = {"foo"} package = Package("foo", 10, mirror) package.sync() assert not os.path.exists( "web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep)) assert mirror.errors
def test__casing__no__affect(self) -> None: mock_config("""\ [mirror] storage-backend = filesystem workers = 2 [plugins] enabled = allowlist_release [allowlist] packages = Foo<=1.2.0 """) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1) pkg._metadata = { "info": { "name": "foo" }, "releases": { "1.2.0": {}, "1.2.1": {} }, } pkg.filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.2.0": {}})
def test__dont__filter__prereleases(self) -> None: mock_config( """\ [plugins] enabled = allowlist_release [allowlist] packages = foo<=1.2.0 """ ) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1) pkg._metadata = { "info": {"name": "foo"}, "releases": { "1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}, "1.2.1": {}, "1.2.2alpha3": {}, "1.2.3rc1": {}, }, } pkg.filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test_latest_releases_keep_stable(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": { "name": "foo", "version": "2.0.0" }, # stable version "releases": { "1.0.0": {}, "1.1.0": {}, "1.1.1": {}, "1.1.2": {}, "1.1.3": {}, "2.0.0": {}, # <= stable version, keep it "2.0.1b1": {}, "2.0.1b2": {}, # <= most recent, keep it }, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg.releases == {"2.0.1b2": {}, "2.0.0": {}}
def test_package_sync_downloads_release_file(mirror, requests): requests.prepare( { "info": { "name": "foo", "version": "0.1" }, "last_serial": 654_321, "releases": { "0.1": [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c" "9e87fb0d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] }, }, 10, ) requests.prepare(b"the release content", 10) mirror.packages_to_sync = {"foo": None} package = Package("foo", 10, mirror) package.sync() assert open("web/packages/any/f/foo/foo.zip").read() == ( "the release content")
def test_package_sync_replaces_mismatching_local_files(mirror, requests): requests.prepare( { "releases": { "0.1": [{ "url": "https://pypi.example.com/packages/any/f/foo/foo.zip", "filename": "foo.zip", "digests": { "md5": "b6bcb391b040c4468262706faf9d3cce", "sha256": ("02db45ea4e09715fbb1ed0fef30d7324db07c" "9e87fb0d4e5470a3e4e878bd8cd"), }, "md5_digest": "b6bcb391b040c4468262706faf9d3cce", }] } }, 10, ) requests.prepare(b"the release content", 10) os.makedirs("web/packages/any/f/foo") with open("web/packages/any/f/foo/foo.zip", "wb") as f: f.write(b"this is not the release content") mirror.packages_to_sync = {"foo": None} package = Package("foo", 10, mirror) package.sync() assert open("web/packages/any/f/foo/foo.zip").read() == ( "the release content")
def test_survives_exceptions_from_record_finished_package(mirror, requests): def record_finished_package(name): import errno raise IOError(errno.EBADF, "Some transient error?") requests.prepare({"releases": {}}, "10") mirror.packages_to_sync = {"Foo": 10} mirror.record_finished_package = record_finished_package package = Package("Foo", 10, mirror) package.sync() assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> </body> </html>\ """) assert mirror.errors
async def test_keep_index_versions_stores_different_prior_versions( mirror: Mirror, ) -> None: simple_path = Path("web/simple/foo") versions_path = simple_path / "versions" mirror.packages_to_sync = {"foo": 1} mirror.keep_index_versions = 2 with freeze_time("2018-10-27"): package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors mirror.packages_to_sync = {"foo": 1} with freeze_time("2018-10-28"): package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors version_files = sorted(os.listdir(versions_path)) assert len(version_files) == 2 assert version_files[0].startswith("index_1_2018-10-27") assert version_files[1].startswith("index_1_2018-10-28") link_path = simple_path / "index.html" assert os.path.islink(link_path) assert os.path.basename(os.readlink(str(link_path))) == version_files[1]
def test_package_sync_with_canonical_simple_page(mirror, requests): requests.prepare( { "info": { "name": "foo" }, "last_serial": 654_321, "releases": {} }, "10") mirror.packages_to_sync = {"Foo": 10} package = Package("Foo", 10, mirror) package.sync() # Cross-check that simple directory hashing is disabled. assert not os.path.exists("web/simple/f/foo/index.html") assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> </body> </html> <!--SERIAL 654321-->\ """)
def test_save_json_metadata(mirror: Mirror, package_json: Dict[str, Any]) -> None: package = Package("foo", 11, mirror) package.json_file.parent.mkdir(parents=True) package.json_pypi_symlink.parent.mkdir(parents=True) package.json_pypi_symlink.symlink_to(Path(gettempdir())) assert package.save_json_metadata(package_json)
def test__filter__matches__release(self) -> None: with open(Path(self.tempdir.name) / "requirements.txt", "w") as fh: fh.write( """\ # This is needed for workshop 1 # foo==1.2.0 # via -r requirements.in """ ) mock_config( f"""\ [plugins] enabled = project_requirements project_requirements_pinned [allowlist] requirements_path = {self.tempdir.name} requirements = requirements.txt """ ) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1) pkg._metadata = { "info": {"name": "foo"}, "releases": {"1.2.0": {}, "1.2.1": {}}, } pkg.filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual({"1.2.0": {}}, pkg.releases)
def test_latest_releases_uninitialized(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": { "name": "foo", "version": "2.0.0" }, "releases": { "1.0.0": {}, "1.1.0": {}, "1.1.1": {}, "1.1.2": {}, "1.1.3": {}, "2.0.0": {}, }, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg.releases == { "1.0.0": {}, "1.1.0": {}, "1.1.1": {}, "1.1.2": {}, "1.1.3": {}, "2.0.0": {}, }
def test_package_sync_with_normalized_simple_page(mirror, requests): requests.prepare( { "info": { "name": "foo" }, "last_serial": 654_321, "releases": {} }, "10") mirror.packages_to_sync = {"Foo.bar-thing_other": 10} package = Package("Foo.bar-thing_other", 10, mirror) package.sync() # PEP 503 normalization assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> </body> </html> <!--SERIAL 654321-->\ """) # Legacy partial normalization as implemented by pip prior to 8.1.2 assert (open("web/simple/foo.bar-thing-other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> </body> </html> <!--SERIAL 654321-->\ """) # Legacy unnormalized as implemented by pip prior to 6.0 assert (open("web/simple/Foo.bar-thing_other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> </body> </html> <!--SERIAL 654321-->\ """)
def test_gen_data_requires_python(mirror: Mirror) -> None: fake_no_release: Dict[str, str] = {} fake_release = {"requires_python": ">=3.6"} package = Package("foo", 10, mirror) assert package.gen_data_requires_python(fake_no_release) == "" assert (package.gen_data_requires_python(fake_release) == ' data-requires-python=">=3.6"')
def test_gen_data_requires_python(mirror, requests): fake_no_release = {} fake_release = {"requires_python": ">=3.6"} package = Package("foo", 10, mirror) assert package.gen_data_requires_python(fake_no_release) == "" assert (package.gen_data_requires_python(fake_release) == ' data-requires-python=">=3.6"')
def test_package_sync_with_error_keeps_it_on_todo_list(mirror, requests): mirror.master.package_releases = mock.Mock() mirror.master.package_releases.return_value = ["0.1"] mirror.master.release_urls = mock.Mock() mirror.master.release_urls.return_value = [] requests.side_effect = Exception mirror.packages_to_sync = {"foo"} package = Package("foo", 10, mirror) package.sync() assert "foo" in mirror.packages_to_sync
def setUp_mirror(self) -> None: self.mirror = Mirror(self.mirror_path, Master(url="https://foo.bar.com")) pkg = Package("foobar", 1, self.mirror) pkg._metadata = { "info": { "name": "foobar", "version": "1.0" }, "releases": mock.Mock(), } self.pkgs.append(pkg)
def test_plugin_check_match(self): _mock_config(self.config_contents) bandersnatch.filter.filter_release_plugins() mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}} pkg._filter_releases() assert pkg.releases == {"foo-1.2.0": {}}
def test_plugin_check_match(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": {"name": "foo", "version": "foo-1.2.0"}, "releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg.releases == {"foo-1.2.0": {}}
async def test_cleanup_non_pep_503_paths(mirror): raw_package_name = "CatDogPython69" package = Package(raw_package_name, 11, mirror) await package.cleanup_non_pep_503_paths() # Create a non normailized directory touch_files( [package.mirror.webdir / "simple" / raw_package_name / "index.html"]) package.cleanup = True with mock.patch("bandersnatch.package.rmtree") as mocked_rmtree: await package.cleanup_non_pep_503_paths() assert mocked_rmtree.call_count == 1
def setUp_mirror(self) -> None: self.master = Master(url="https://foo.bar.com") self.mirror = BandersnatchMirror(self.mirror_path, self.master, self.backend) pkg = Package("foobar", serial=1) pkg._metadata = { "info": { "name": "foobar", "version": "1.0" }, "releases": mock.Mock(), } self.pkgs.append(pkg)
async def test_package_sync_gives_up_after_3_stale_responses(caplog, mirror): mirror.master.package_releases = mock.Mock() mirror.master.package_releases.return_value = ["0.1"] mirror.master.release_urls = mock.Mock() mirror.master.release_urls.return_value = [] pkg_name = "foo" package = Package(pkg_name, 11, mirror) package.sleep_on_stale = 0 await package.sync() assert package.tries == 3 assert mirror.errors assert "not updating. Giving up" in caplog.text
def test_sync_keeps_superfluous_files_on_nondeleting_mirror(mirror, requests): test_files = [Path("web/packages/2.4/f/foo/foo.zip")] touch_files(test_files) mirror.master.package_releases = mock.Mock() mirror.master.package_releases.return_value = ["0.1"] mirror.master.release_urls = mock.Mock() mirror.master.release_urls.return_value = [] mirror.packages_to_sync = {"foo"} package = Package("foo", 10, mirror) package.sync() assert test_files[0].exists()
def test_latest_releases_ensure_reusable(self) -> None: """ Tests the filter multiple times to ensure no state is preserved and thus is reusable between packages """ mock_config(self.config_contents) mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com")) pkg1 = Package("foo", 1) pkg1._metadata = { "info": { "name": "foo", "version": "2.0.0" }, "releases": { "0.1.1": {}, "0.1.2": {}, "0.1.3": {}, "1.0.0": {}, "1.1.0": {}, "1.2.0": {}, "2.0.0": {}, }, } pkg2 = Package("bar", 1) pkg2._metadata = { "info": { "name": "bar", "version": "0.3.0" }, "releases": { "0.1.0": {}, "0.1.1": {}, "0.1.2": {}, "0.1.3": {}, "0.1.4": {}, "0.1.5": {}, "0.2.0": {}, "0.3.0": {}, }, } pkg1.filter_all_releases(mirror.filters.filter_release_plugins()) pkg2.filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg1.releases == {"1.2.0": {}, "2.0.0": {}} assert pkg2.releases == {"0.2.0": {}, "0.3.0": {}}