async def test_keep_index_versions_stores_different_prior_versions( mirror: Mirror, ) -> None: simple_path = Path("web/simple/foo") versions_path = simple_path / "versions" mirror.packages_to_sync = {"foo": 1} mirror.keep_index_versions = 2 with freeze_time("2018-10-27"): package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors mirror.packages_to_sync = {"foo": 1} with freeze_time("2018-10-28"): package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors version_files = sorted(os.listdir(versions_path)) assert len(version_files) == 2 assert version_files[0].startswith("index_1_2018-10-27") assert version_files[1].startswith("index_1_2018-10-28") link_path = simple_path / "index.html" assert os.path.islink(link_path) assert os.path.basename(os.readlink(str(link_path))) == version_files[1]
async def test_package_sync_downloads_release_file(mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": ""} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert open("web/packages/any/f/foo/foo.zip").read() == ""
async def test_package_sync_handles_non_pep_503_in_packages_to_sync(master): with TemporaryDirectory() as td: mirror = Mirror(td, master, stop_on_error=True) mirror.packages_to_sync = {"Foo": None} package = Package("Foo", 1, mirror) await package.sync() assert not mirror.errors
async def test_survives_exceptions_from_record_finished_package( mirror: Mirror) -> None: def record_finished_package(name: str) -> NoReturn: import errno raise OSError(errno.EBADF, "Some transient error?") mirror.packages_to_sync = {"Foo": 1} mirror.record_finished_package = record_finished_package # type: ignore package = Package("Foo", 1, mirror) await package.sync(mirror.filters) assert (Path("web/simple/foo/index.html").open().read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS)) assert mirror.errors
async def test_package_sync_simple_page_root_uri(mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": 1} mirror.root_uri = "https://files.pythonhosted.org" package = Package("foo", 1, mirror) await package.sync(mirror.filters) mirror.root_uri = None expected_root_uri_hrefs = ( '<a href="https://files.pythonhosted.org/packages/2.7/f/foo/foo.whl#sha256=e3b' + '0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">foo.whl</a>' + '<br/>\n <a href="https://files.pythonhosted.org/packages/any/f/foo/foo.' + "zip#sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + '">foo.zip</a><br/>') assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(expected_root_uri_hrefs))
def test_plugin_check_match(self) -> None: mock_config(self.config_contents) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo-good": "", "foo-evil": "", "foo-neutral": ""} mirror._filter_packages() assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
async def test_package_download_rejects_non_package_directory_links( mirror: Mirror, ) -> None: mirror.packages_to_sync = {"foo"} # type: ignore package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert mirror.errors assert "foo" in mirror.packages_to_sync assert not os.path.exists("web/foo/bar/foo/foo.zip")
async def test_package_sync_handles_non_pep_503_in_packages_to_sync( master: Master, ) -> None: with TemporaryDirectory() as td: mirror = Mirror(Path(td), master, stop_on_error=True) mirror.packages_to_sync = {"Foo": ""} package = Package("Foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors
async def test_sync_incorrect_download_with_old_serials_retries( mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": 1} package = Package("foo", 2, mirror) await package.sync(mirror.filters) assert not Path("web/packages/any/f/foo/foo.zip").exists() assert mirror.errors
async def test_package_sync_with_error_keeps_it_on_todo_list( mirror: Mirror) -> None: # Make packages_to_sync to generate an error mirror.packages_to_sync = {"foo"} # type: ignore package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert mirror.errors assert "foo" in mirror.packages_to_sync
def test_plugin_check_match(self): _mock_config(self.config_contents) bandersnatch.filter.filter_release_plugins() mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo-good": {}, "foo-evil": {}, "foo-neutral": {}} mirror._filter_packages() assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
async def test_sync_keeps_superfluous_files_on_nondeleting_mirror( mirror: Mirror, ) -> None: test_files = [Path("web/packages/2.4/f/foo/foo.zip")] touch_files(test_files) mirror.packages_to_sync = {"foo": ""} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert test_files[0].exists()
async def test_package_sync_replaces_mismatching_local_files( mirror: Mirror) -> None: test_files = [Path("web/packages/any/f/foo/foo.zip")] touch_files(test_files) with test_files[0].open("wb") as f: f.write(b"this is not the release content") mirror.packages_to_sync = {"foo": ""} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert test_files[0].open("r").read() == ""
def test__filter__nomatch_package(self) -> None: mock_config( """\ [blacklist] plugins = blacklist_project packages = foo """ ) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo2": ""} mirror._filter_packages() self.assertIn("foo2", mirror.packages_to_sync.keys())
async def test_keep_index_versions_stores_one_prior_version( mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": ""} mirror.keep_index_versions = 1 package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors simple_path = Path("web/simple/foo") versions_path = simple_path / "versions" version_files = os.listdir(versions_path) assert len(version_files) == 1 assert version_files[ 0] == f"index_{package.serial}_{make_time_stamp()}.html" link_path = simple_path / "index.html" assert link_path.is_symlink() assert os.path.basename(os.readlink(str(link_path))) == version_files[0]
async def test_package_sync_does_not_touch_existing_local_file( mirror: Mirror) -> None: pkg_file_path_str = "web/packages/any/f/foo/foo.zip" pkg_file_path = Path(pkg_file_path_str) touch_files((pkg_file_path, )) with pkg_file_path.open("w") as f: f.write("") old_stat = pkg_file_path.stat() mirror.packages_to_sync = {"foo": 1} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors # Use Pathlib + create a new object to ensure no caching # Only compare the relevant stat fields assert old_stat.st_mtime == Path(pkg_file_path_str).stat().st_mtime assert old_stat.st_ctime == Path(pkg_file_path_str).stat().st_ctime
def test__filter__nomatch_package(self): with open("test.conf", "w") as testconfig_handle: testconfig_handle.write("""\ [blacklist] plugins = blacklist_project packages = foo """) instance = BandersnatchConfig() instance.config_file = "test.conf" instance.load_configuration() mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo2": {}} mirror._filter_packages() self.assertIn("foo2", mirror.packages_to_sync.keys())
def test__filter__matches__package(self) -> None: mock_config("""\ [mirror] storage-backend = filesystem [plugins] enabled = whitelist_project [whitelist] packages = foo """) mirror = Mirror(Path("."), Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": ""} mirror._filter_packages() self.assertIn("foo", mirror.packages_to_sync.keys())
async def test_package_sync_simple_page_with_files(mirror: Mirror) -> None: mirror.packages_to_sync = {"foo": 1} package = Package("foo", 1, mirror) await package.sync(mirror.filters) assert not mirror.errors assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for foo</title> </head> <body> <h1>Links for foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
async def test_package_sync_with_canonical_simple_page_with_hash( mirror_hash_index: Mirror, ) -> None: mirror_hash_index.packages_to_sync = {"Foo": 1} package = Package("Foo", 1, mirror_hash_index) await package.sync(mirror_hash_index.filters) assert not os.path.exists("web/simple/foo/index.html") assert (open("web/simple/f/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
def test_mirror_filter_packages_match(tmpdir): """ Packages that exist in the blacklist should be removed from the list of packages to sync. """ test_configuration = """\ [blacklist] packages = example1 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") for plugin in filter_project_plugins(): plugin.initialize_plugin() m = Mirror(str(tmpdir), mock.Mock()) m.packages_to_sync = {"example1": None, "example2": None} m._filter_packages() assert "example1" not in m.packages_to_sync.keys()
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir): """ Package lines with a PEP440 spec on them should not be filtered from the list of packages. """ test_configuration = """\ [blacklist] packages = example3>2.0.0 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") for plugin in filter_project_plugins(): plugin.initialize_plugin() m = Mirror(str(tmpdir), mock.Mock()) m.packages_to_sync = {"example1": None, "example3": None} m._filter_packages() assert "example3" in m.packages_to_sync.keys()
async def test_package_sync_with_normalized_simple_page( mirror: Mirror) -> None: mirror.packages_to_sync = {"Foo.bar-thing_other": 1} package = Package("Foo.bar-thing_other", 1, mirror) await package.sync(mirror.filters) # PEP 503 normalization assert (open("web/simple/foo-bar-thing-other/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo.bar-thing_other</title> </head> <body> <h1>Links for Foo.bar-thing_other</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
def test__filter__matches__package(self): with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write("""\ [plugins] enabled = whitelist_project [whitelist] packages = foo """) instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": {}} mirror._filter_packages() self.assertIn("foo", mirror.packages_to_sync.keys())
async def test_package_sync_with_canonical_simple_page(mirror: Mirror) -> None: mirror.packages_to_sync = {"Foo": 1} package = Package("Foo", 1, mirror) await package.sync(mirror.filters) # Cross-check that simple directory hashing is disabled. assert not os.path.exists("web/simple/f/foo/index.html") assert (open("web/simple/foo/index.html").read() == """\ <!DOCTYPE html> <html> <head> <title>Links for Foo</title> </head> <body> <h1>Links for Foo</h1> {} </body> </html> <!--SERIAL 654321-->\ """.format(EXPECTED_REL_HREFS))
def test_mirror_filter_packages_match(tmpdir: Path) -> None: """ Packages that exist in the blacklist should be removed from the list of packages to sync. """ test_configuration = """\ [plugins] enabled = blacklist_project [blacklist] packages = example1 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") m = Mirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example2": ""} m._filter_packages() assert "example1" not in m.packages_to_sync.keys()
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None: """ Package lines with a PEP440 spec on them should not be filtered from the list of packages. """ test_configuration = """\ [plugins] enable = blacklist_project [blacklist] packages = example3>2.0.0 """ Singleton._instances = {} with open("test.conf", "w") as testconfig_handle: testconfig_handle.write(test_configuration) BandersnatchConfig("test.conf") m = Mirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example3": ""} m._filter_packages() assert "example3" in m.packages_to_sync.keys()
def test__filter__nomatch_package(self): _mock_config( """\ [mirror] storage-backend = filesystem [plugins] enabled = whitelist_project [whitelist] packages = foo """ ) mirror = Mirror(".", Master(url="https://foo.bar.com")) mirror.packages_to_sync = {"foo": {}, "foo2": {}} mirror._filter_packages() self.assertIn("foo", mirror.packages_to_sync.keys()) self.assertNotIn("foo2", mirror.packages_to_sync.keys())