def test_task_lister_gitlab(
task_name,
incremental,
swh_scheduler_celery_app,
swh_scheduler_celery_worker,
mocker,
):
stats = ListerStats(pages=10, origins=200)
mock_lister = mocker.patch("swh.lister.gitlab.tasks.GitLabLister")
mock_lister.from_configfile.return_value = mock_lister
mock_lister.run.return_value = ListerStats(pages=10, origins=200)
kwargs = dict(url="https://gitweb.torproject.org/")
res = swh_scheduler_celery_app.send_task(
f"swh.lister.gitlab.tasks.{task_name}",
kwargs=kwargs,
)
assert res
res.wait()
assert res.successful()
mock_lister.from_configfile.assert_called_once_with(
incremental=incremental, **kwargs)
mock_lister.run.assert_called_once_with()
assert res.result == stats.dict()
def test_crates_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
# setup the mocked CratesLister
lister = mocker.patch("swh.lister.crates.tasks.CratesLister")
lister.from_configfile.return_value = lister
stats = ListerStats(pages=42, origins=42)
lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.crates.tasks.CratesListerTask")
assert res
res.wait()
assert res.successful()
assert res.result == stats.dict()
lister.from_configfile.assert_called_once_with()
lister.run.assert_called_once_with()
def test_incremental_listing(swh_scheduler_celery_app,
swh_scheduler_celery_worker, mocker):
stats = ListerStats(pages=1, origins=90)
mock_lister = mocker.patch(lister_module)
mock_lister.from_configfile.return_value = mock_lister
mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.sourceforge.tasks.IncrementalSourceForgeLister")
assert res
res.wait()
assert res.successful()
mock_lister.from_configfile.assert_called_once()
mock_lister.run.assert_called_once()
assert res.result == stats.dict()
def test_launchpad_full_listing_task(swh_scheduler_celery_app,
swh_scheduler_celery_worker, mocker):
lister = mocker.patch("swh.lister.launchpad.tasks.LaunchpadLister")
lister.from_configfile.return_value = lister
stats = ListerStats(pages=1, origins=28000)
lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.launchpad.tasks.FullLaunchpadLister")
assert res
res.wait()
assert res.successful()
assert res.result == stats.dict()
lister.from_configfile.assert_called_once_with()
lister.run.assert_called_once_with()
def test_full_lister_task(
swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
stats = ListerStats(pages=10, origins=900)
mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
mock_lister.from_configfile.return_value = mock_lister
mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask")
assert res
res.wait()
assert res.successful()
mock_lister.from_configfile.assert_called_once_with(incremental=False)
mock_lister.run.assert_called_once_with()
assert res.result == stats.dict()
def test_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
lister = mocker.patch("swh.lister.packagist.tasks.PackagistLister")
lister.from_configfile.return_value = lister
stats = ListerStats(pages=1, origins=286500)
lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.packagist.tasks.PackagistListerTask"
)
assert res
res.wait()
assert res.successful()
assert res.result == stats.dict()
lister.from_configfile.assert_called_once_with()
lister.run.assert_called_once_with()
def test_phabricator_lister_task(lister, swh_scheduler_celery_app,
swh_scheduler_celery_worker):
# setup the mocked PhabricatorLister
lister.from_configfile.return_value = lister
lister_stats = ListerStats(pages=2, origins=200)
lister.run.return_value = lister_stats
task_params = {
"url": "https://forge.softwareheritage.org",
"instance": "swh",
"api_token": None,
}
res = swh_scheduler_celery_app.send_task(
"swh.lister.phabricator.tasks.FullPhabricatorLister",
kwargs=task_params)
assert res
res.wait()
assert res.successful()
assert res.result == lister_stats.dict()
lister.from_configfile.assert_called_once_with(**task_params)
def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
# setup the mocked DebianLister
lister.from_configfile.return_value = lister
stats = ListerStats(pages=12, origins=35618)
lister.run.return_value = stats
kwargs = dict(
mirror_url=
"http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/",
distribution="Ubuntu",
suites=["xenial", "bionic", "focal"],
components=["main", "multiverse", "restricted", "universe"],
)
res = swh_scheduler_celery_app.send_task(
"swh.lister.debian.tasks.DebianListerTask", kwargs=kwargs)
assert res
res.wait()
assert res.successful()
lister.from_configfile.assert_called_once_with(**kwargs)
lister.run.assert_called_once_with()
assert res.result == stats.dict()
