def test_lister_cgit_get_pages_with_pages_and_retry(requests_mock_datadir, requests_mock, datadir, mocker, swh_scheduler): url = "https://git.tizen/cgit/" with open(os.path.join(datadir, "https_git.tizen/cgit,ofs=50"), "rb") as page: requests_mock.get( f"{url}?ofs=50", [ { "content": None, "status_code": 429 }, { "content": None, "status_code": 429 }, { "content": page.read(), "status_code": 200 }, ], ) lister_cgit = CGitLister(swh_scheduler, url=url) mocker.patch.object(lister_cgit._get_and_parse.retry, "sleep") repos: List[List[str]] = list(lister_cgit.get_pages()) flattened_repos = sum(repos, []) # we should have 16 repos (listed on 3 pages) assert len(repos) == 3 assert len(flattened_repos) == 16
def test_lister_cgit_get_pages_with_pages(requests_mock_datadir, swh_scheduler): url = "https://git.tizen/cgit/" lister_cgit = CGitLister(swh_scheduler, url=url) repos: List[List[str]] = list(lister_cgit.get_pages()) flattened_repos = sum(repos, []) # we should have 16 repos (listed on 3 pages) assert len(repos) == 3 assert len(flattened_repos) == 16
def test_lister_cgit_get_pages_one_page(requests_mock_datadir, swh_scheduler): url = "https://git.savannah.gnu.org/cgit/" lister_cgit = CGitLister(swh_scheduler, url=url) repos: List[List[str]] = list(lister_cgit.get_pages()) flattened_repos = sum(repos, []) assert len(flattened_repos) == 977 assert flattened_repos[0][ "url"] == "https://git.savannah.gnu.org/cgit/elisp-es.git" # note the url below is NOT a subpath of /cgit/ assert (flattened_repos[-1]["url"] == "https://git.savannah.gnu.org/path/to/yetris.git") # noqa # note the url below is NOT on the same server assert flattened_repos[-2][ "url"] == "http://example.org/cgit/xstarcastle.git"