def setUp_mirror(self) -> None: self.mirror = Mirror(self.mirror_path, Master(url="https://foo.bar.com")) pkg = Package("foobar", 1, self.mirror) pkg._metadata = { "info": { "name": "foobar", "version": "1.0" }, "releases": mock.Mock(), } self.pkgs.append(pkg)
def test_mirror_filter_packages_match(tmpdir): """ Packages that exist in the blacklist should be removed from the list of packages to sync. """ test_configuration = """\ [blacklist] plugins = blacklist_project packages = example1 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") for plugin in filter_project_plugins(): plugin.initialize_plugin() m = Mirror(str(tmpdir), mock.Mock()) m.packages_to_sync = {"example1": None, "example2": None} m._filter_packages() assert "example1" not in m.packages_to_sync.keys()
def test__filter__matches__package(self): _mock_config( """\ [mirror] storage-backend = filesystem [plugins] enabled = whitelist_project [whitelist] packages = foo """ ) mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": {}} mirror._filter_packages() self.assertIn("foo", mirror.packages_to_sync.keys())
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None: """ Package lines with a PEP440 spec on them should not be filtered from the list of packages. """ test_configuration = """\ [plugins] enable = blacklist_project [blacklist] packages = example3>2.0.0 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") m = Mirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example3": ""} m._filter_packages() assert "example3" in m.packages_to_sync.keys()
async def test_cleanup_non_pep_503_paths(mirror: Mirror) -> None: raw_package_name = "CatDogPython69" package = Package(raw_package_name, 11, mirror) await mirror.cleanup_non_pep_503_paths(package) # Create a non normailized directory touch_files([mirror.webdir / "simple" / raw_package_name / "index.html"]) mirror.cleanup = True with mock.patch("bandersnatch.mirror.rmtree") as mocked_rmtree: await mirror.cleanup_non_pep_503_paths(package) assert mocked_rmtree.call_count == 1
def test_mirror_filter_packages_match(tmpdir: Path) -> None: """ Packages that exist in the blacklist should be removed from the list of packages to sync. """ test_configuration = """\ [plugins] enabled = blacklist_project [blacklist] packages = example1 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") m = Mirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example2": ""} m._filter_packages() assert "example1" not in m.packages_to_sync.keys()
def test_plugin_check_match(self): _mock_config(self.config_contents) bandersnatch.filter.filter_release_plugins() mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}} pkg._filter_releases() assert pkg.releases == {"foo-1.2.0": {}}
def test__filter__matches__package(self): with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write( """\ [plugins] enabled = blacklist_project [blacklist] packages = foo """ ) instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": {}} mirror._filter_packages() self.assertNotIn("foo", mirror.packages_to_sync.keys())
def test_plugin_check_match(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": {"name": "foo", "version": "foo-1.2.0"}, "releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg.releases == {"foo-1.2.0": {}}
async def test_mirror_serial_current_no_sync_of_packages_and_index_page( mirror: Mirror, ) -> None: mirror.master.changed_packages = asynctest.CoroutineMock( # type: ignore return_value={} ) mirror.synced_serial = 1 await mirror.synchronize() assert """\ last-modified""" == utils.find( mirror.webdir, dirs=False )
def test_mirror_generation_4_resets_status_files(tmpdir): with open(str(tmpdir / "generation"), "w") as generation: generation.write("4") with open(str(tmpdir / "status"), "w") as status: status.write("1234") with open(str(tmpdir / "todo"), "w") as status: status.write("asdf") m = Mirror(str(tmpdir), mock.Mock()) assert m.synced_serial == 0 assert not os.path.exists(str(tmpdir / "todo")) assert not os.path.exists(str(tmpdir / "status")) assert open(str(tmpdir / "generation"), "r").read() == "5"
async def test_package_sync_replaces_mismatching_local_files( mirror: Mirror) -> None: test_files = [Path("web/packages/any/f/foo/foo.zip")] touch_files(test_files) with test_files[0].open("wb") as f: f.write(b"this is not the release content") mirror.packages_to_sync = {"foo": ""} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert test_files[0].open("r").read() == ""
def test__filter__nomatch_package(self): with open("test.conf", "w") as testconfig_handle: testconfig_handle.write( """\ [blacklist] plugins = whitelist_project [whitelist] packages = foo """ ) instance = BandersnatchConfig() instance.config_file = "test.conf" instance.load_configuration() mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": {}, "foo2": {}} mirror._filter_packages() self.assertIn("foo", mirror.packages_to_sync.keys()) self.assertNotIn("foo2", mirror.packages_to_sync.keys())
def test_find_package_indexes_in_dir_threaded(mirror: Mirror) -> None: directories = ( "web/simple/peerme", "web/simple/click", "web/simple/zebra", "web/simple/implicit", "web/simple/pyaib", "web/simple/setuptools", ) with TemporaryDirectory() as td: # Create local mirror first so we '_bootstrap' mirror_base = Path(td) local_mirror = Mirror(mirror_base, mirror.master, stop_on_error=True) # Create fake file system objects for directory in directories: (mirror_base / directory).mkdir(parents=True, exist_ok=True) with (mirror_base / "web/simple/index.html").open("w") as index: index.write("<html></html>") packages = local_mirror.find_package_indexes_in_dir(mirror_base / "web/simple") assert "index.html" not in packages # This should never be in the list assert len(packages) == 6 # We expect 6 packages with 6 dirs created assert packages[0] == "click" # Check sorted - click should be first
def test__filter__pyversions(self): instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() with open(TEST_JSON) as json_file: package_json = json.load(json_file) mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.info = package_json["info"] pkg.releases = package_json["releases"] pkg._filter_releases() self.assertEqual(set([v['python_version'] for v in pkg.releases["0.4.3"]]), set(['cp35', 'cp36']))
async def test_package_sync_does_not_touch_existing_local_file( mirror: Mirror) -> None: pkg_file_path_str = "web/packages/any/f/foo/foo.zip" pkg_file_path = Path(pkg_file_path_str) touch_files((pkg_file_path, )) with pkg_file_path.open("w") as f: f.write("") old_stat = pkg_file_path.stat() mirror.packages_to_sync = {"foo": 1} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors # Use Pathlib + create a new object to ensure no caching # Only compare the relevant stat fields assert old_stat.st_mtime == Path(pkg_file_path_str).stat().st_mtime assert old_stat.st_ctime == Path(pkg_file_path_str).stat().st_ctime
async def test_mirror_sync_package_error_no_early_exit(mirror: Mirror) -> None: mirror.master.all_packages = asynctest.CoroutineMock( # type: ignore return_value={"foo": 1} ) mirror.errors = True changed_packages = await mirror.synchronize() expected = """\ .lock generation todo web{0}packages{0}2.7{0}f{0}foo{0}foo.whl web{0}packages{0}any{0}f{0}foo{0}foo.zip web{0}simple{0}foo{0}index.html web{0}simple{0}index.html""".format( sep ) if WINDOWS: expected = expected.replace(".lock\n", "") assert expected == utils.find(mirror.homedir, dirs=False) assert ( open("web{0}simple{0}index.html".format(sep)).read() == """\ <!DOCTYPE html> <html> <head> <title>Simple Index</title> </head> <body> <a href="foo/">foo</a><br/> </body> </html>""" ) assert open("todo").read() == "1\n" # Check the returned dict is accurate expected_dict = { "foo": { "web{0}packages{0}2.7{0}f{0}foo{0}foo.whl".format(sep), "web{0}packages{0}any{0}f{0}foo{0}foo.zip".format(sep), } } assert changed_packages == expected_dict
async def test_package_sync_simple_page_with_files(mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": 1} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
def test_plugin_check_match(self): _mock_config(self.config_contents) bandersnatch.filter.filter_release_plugins() mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.info = {"name": "foo", "version": "1.2.0"} pkg.releases = { "1.2.0alpha1": {}, "1.2.0a2": {}, "1.2.0beta1": {}, "1.2.0b2": {}, "1.2.0rc1": {}, "1.2.0": {}, } pkg._filter_releases() assert pkg.releases == {"1.2.0": {}}
async def test_package_sync_with_normalized_simple_page( mirror: Mirror) -> None: mirror.packages_to_sync = {"Foo.bar-thing_other": 1} package = Package("Foo.bar-thing_other", 1, mirror) await package.sync(mirror.filters) # PEP 503 normalization assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page_with_hash( mirror_hash_index: Mirror, ) -> None: mirror_hash_index.packages_to_sync = {"Foo": 1} package = Package("Foo", 1, mirror_hash_index) await package.sync(mirror_hash_index.filters) assert not os.path.exists("web/simple/foo/index.html") assert (open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page(mirror: Mirror) -> None: mirror.packages_to_sync = {"Foo": 1} package = Package("Foo", 1, mirror) await package.sync(mirror.filters) # Cross-check that simple directory hashing is disabled. assert not os.path.exists("web/simple/f/foo/index.html") assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
def test_latest_releases_keep_latest(self): _mock_config(self.config_contents) bandersnatch.filter.filter_release_plugins() mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.info = {"name": "foo", "version": "2.0.0"} pkg.releases = { "1.0.0": {}, "1.1.0": {}, "1.1.1": {}, "1.1.2": {}, "1.1.3": {}, "2.0.0": {}, } pkg._filter_releases() assert pkg.releases == {"1.1.3": {}, "2.0.0": {}}
def test__filter__matches__release(self): with open("test.conf", "w") as testconfig_handle: testconfig_handle.write("""\ [blacklist] plugins = blacklist_release packages = foo==1.2.0 """) instance = BandersnatchConfig() instance.config_file = "test.conf" instance.load_configuration() mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg.releases = {"1.2.0": {}, "1.2.1": {}} pkg._filter_releases() self.assertEqual(pkg.releases, {"1.2.1": {}})
async def test_keep_index_versions_removes_old_versions( mirror: Mirror) -> None: simple_path = Path("web/simple/foo/") versions_path = simple_path / "versions" versions_path.mkdir(parents=True) (versions_path / "index_1_2018-10-26T000000Z.html").touch() (versions_path / "index_1_2018-10-27T000000Z.html").touch() mirror.keep_index_versions = 2 with freeze_time("2018-10-28"): package = Package("foo", 1, mirror) await package.sync(mirror.filters) version_files = sorted(f for f in versions_path.iterdir()) assert len(version_files) == 2 assert version_files[0].name.startswith("index_1_2018-10-27") assert version_files[1].name.startswith("index_1_2018-10-28") link_path = simple_path / "index.html" assert link_path.is_symlink() assert os.path.basename(os.readlink( str(link_path))) == version_files[1].name
def test__filter__matches__release(self): with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write( """\ [blacklist] plugins = safety_db_release """ ) instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() self._dump_config(instance) mirror = Mirror(".", Master(url="https://foo.bar.com")) pkg = Package("aiohttp", 1, mirror) pkg.info = {"name": "aiohttp"} pkg.releases = {"0.16.3": {}, "0.16.0": {}, "0.15.1": {}} pkg._filter_releases() self.assertEqual(pkg.releases, {"0.16.3": {}})
def test__filter__matches__release(self) -> None: mock_config( """\ [plugins] enabled = blacklist_release [blacklist] packages = foo==1.2.0 """ ) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": {"name": "foo"}, "releases": {"1.2.0": {}, "1.2.1": {}}, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) self.assertEqual(pkg.releases, {"1.2.1": {}})
def test_latest_releases_keep_latest(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) pkg = Package("foo", 1, mirror) pkg._metadata = { "info": { "name": "foo", "version": "2.0.0" }, "releases": { "1.0.0": {}, "1.1.0": {}, "1.1.1": {}, "1.1.2": {}, "1.1.3": {}, "2.0.0": {}, }, } pkg._filter_all_releases(mirror.filters.filter_release_plugins()) assert pkg.releases == {"1.1.3": {}, "2.0.0": {}}
def mirror_hash_index(tmpdir, master, monkeypatch): monkeypatch.chdir(tmpdir) from bandersnatch.mirror import Mirror return Mirror(str(tmpdir), master, hash_index=True)
def mirror(tmpdir, master, monkeypatch): monkeypatch.chdir(tmpdir) from bandersnatch.mirror import Mirror return Mirror(str(tmpdir), master)