def test_search_in_hdx(self, configuration, search):
resources = Resource.search_in_hdx('name:ACLED')
assert len(resources) == 4
resources = Resource.search_in_hdx('name:ajyhgr')
assert len(resources) == 0
with pytest.raises(HDXError):
Resource.search_in_hdx('fail')
def test_search_in_hdx(self, configuration, search):
resources = Resource.search_in_hdx('name:ACLED')
assert len(resources) == 4
resources = Resource.search_in_hdx('name:ajyhgr')
assert len(resources) == 0
with pytest.raises(HDXError):
Resource.search_in_hdx('fail')
def main(configuration):
resources = Resource.search_in_hdx(configuration, 'name:')
last_modified_check = list()
for resource in resources:
resource_id = resource['id']
url = resource['url']
if 'data.humdata.org' in url or 'manage.hdx.rwlabs.org' in url or 'proxy.hxlstandard.org' in url or \
'scraperwiki.com' in url or 'ourairports.com' in url:
continue
last_modified_check.append((url, resource_id))
last_modified_check = sorted(
last_modified_check)[:NUMBER_OF_URLS_TO_PROCESS]
# run_grequests(last_modified_check)
run_aiohttp(last_modified_check)