def test_search_in_hdx(self, configuration, search): resources = Resource.search_in_hdx('name:ACLED') assert len(resources) == 4 resources = Resource.search_in_hdx('name:ajyhgr') assert len(resources) == 0 with pytest.raises(HDXError): Resource.search_in_hdx('fail')
def test_search_in_hdx(self, configuration, search): resources = Resource.search_in_hdx('name:ACLED') assert len(resources) == 4 resources = Resource.search_in_hdx('name:ajyhgr') assert len(resources) == 0 with pytest.raises(HDXError): Resource.search_in_hdx('fail')
def main(configuration): resources = Resource.search_in_hdx(configuration, 'name:') last_modified_check = list() for resource in resources: resource_id = resource['id'] url = resource['url'] if 'data.humdata.org' in url or 'manage.hdx.rwlabs.org' in url or 'proxy.hxlstandard.org' in url or \ 'scraperwiki.com' in url or 'ourairports.com' in url: continue last_modified_check.append((url, resource_id)) last_modified_check = sorted( last_modified_check)[:NUMBER_OF_URLS_TO_PROCESS] # run_grequests(last_modified_check) run_aiohttp(last_modified_check)