def test_lister_cgit_get_pages_with_pages_and_retry(requests_mock_datadir,
requests_mock, datadir,
mocker, swh_scheduler):
url = "https://git.tizen/cgit/"
with open(os.path.join(datadir, "https_git.tizen/cgit,ofs=50"),
"rb") as page:
requests_mock.get(
f"{url}?ofs=50",
[
{
"content": None,
"status_code": 429
},
{
"content": None,
"status_code": 429
},
{
"content": page.read(),
"status_code": 200
},
],
)
lister_cgit = CGitLister(swh_scheduler, url=url)
mocker.patch.object(lister_cgit._get_and_parse.retry, "sleep")
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
# we should have 16 repos (listed on 3 pages)
assert len(repos) == 3
assert len(flattened_repos) == 16
def test_lister_cgit_get_pages_with_pages(requests_mock_datadir,
swh_scheduler):
url = "https://git.tizen/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
# we should have 16 repos (listed on 3 pages)
assert len(repos) == 3
assert len(flattened_repos) == 16
def test_lister_cgit_get_pages_one_page(requests_mock_datadir, swh_scheduler):
url = "https://git.savannah.gnu.org/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
assert len(flattened_repos) == 977
assert flattened_repos[0][
"url"] == "https://git.savannah.gnu.org/cgit/elisp-es.git"
# note the url below is NOT a subpath of /cgit/
assert (flattened_repos[-1]["url"] ==
"https://git.savannah.gnu.org/path/to/yetris.git") # noqa
# note the url below is NOT on the same server
assert flattened_repos[-2][
"url"] == "http://example.org/cgit/xstarcastle.git"