def main(save, use_saved, **ignore):
"""Generate dataset and create it in HDX"""
with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'),
extra_params_lookup=lookup) as token_downloader:
configuration = Configuration.read()
with Download() as downloader:
folder = temp_dir(lookup)
retriever = Retrieve(downloader, folder, 'saved_data', folder,
save, use_saved)
wfp = WFPFood(configuration, token_downloader, retriever)
countries = wfp.get_countries()
logger.info('Number of country datasets to upload: %d' %
len(countries))
wfp.build_mappings()
for info, country in progress_storing_tempdir(
lookup, countries, 'iso3'):
dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase(
country['iso3'], info['folder'])
if dataset:
dataset.update_from_yaml()
dataset['notes'] = dataset[
'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[
'name']
dataset.generate_resource_view(-1,
indicators=qc_indicators)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: WFP Food Prices',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def test_setup_stream(self, fixtureurl, fixturenotexistsurl, getfixtureurl,
postfixtureurl):
with pytest.raises(DownloadError), Download() as downloader:
downloader.setup('NOTEXIST://NOTEXIST.csv')
with pytest.raises(DownloadError), Download() as downloader:
downloader.setup(fixturenotexistsurl)
with Download() as downloader:
downloader.setup(fixtureurl)
headers = downloader.response.headers
assert headers['Content-Length'] == '728'
with Download() as downloader:
downloader.setup(postfixtureurl, post=True)
headers = downloader.response.headers
assert bool(re.match(r'3[56]\d',
headers['Content-Length'])) is True
downloader.setup('%s?id=10&lala=a' % getfixtureurl,
post=False,
parameters=OrderedDict([('b', '4'), ('d', '3')]))
assert downloader.get_json()['args'] == OrderedDict([('b', '4'),
('d', '3'),
('id', '10'),
('lala', 'a')
])
downloader.setup('%s?id=3&lala=b' % postfixtureurl,
post=True,
parameters=OrderedDict([('a', '3'), ('c', '2')]))
assert downloader.get_json()['form'] == OrderedDict([('a', '3'),
('c', '2'),
('id', '3'),
('lala', 'b')
])
def test_setup_stream(self, fixtureurl, fixturenotexistsurl, getfixtureurl,
postfixtureurl):
with pytest.raises(DownloadError), Download() as downloader:
downloader.setup("NOTEXIST://NOTEXIST.csv")
with pytest.raises(DownloadError), Download() as downloader:
downloader.setup(fixturenotexistsurl)
with Download() as downloader:
downloader.setup(fixtureurl)
headers = downloader.response.headers
assert bool(re.match(r"7\d\d", headers["Content-Length"])) is True
with Download() as downloader:
downloader.setup(postfixtureurl, post=True)
headers = downloader.response.headers
assert bool(re.match(r"4\d\d", headers["Content-Length"])) is True
downloader.setup(
f"{getfixtureurl}?id=10&lala=a",
post=False,
parameters=OrderedDict([("b", "4"), ("d", "3")]),
)
assert list(downloader.get_json()["args"].items()) == list(
OrderedDict([("b", "4"), ("d", "3"), ("id", "10"),
("lala", "a")]).items())
downloader.setup(
f"{postfixtureurl}?id=3&lala=b",
post=True,
parameters=OrderedDict([("a", "3"), ("c", "2")]),
)
assert list(downloader.get_json()["form"].items()) == list(
OrderedDict([("a", "3"), ("c", "2"), ("id", "3"),
("lala", "b")]).items())
def test_download(self, fixtureurl, fixturenotexistsurl, getfixtureurl,
postfixtureurl):
with pytest.raises(DownloadError), Download() as downloader:
downloader.download('NOTEXIST://NOTEXIST.csv')
with pytest.raises(DownloadError), Download() as downloader:
downloader.download(fixturenotexistsurl)
with Download() as downloader:
result = downloader.download(fixtureurl)
assert result.headers['Content-Length'] == '728'
downloader.download('%s?id=10&lala=a' % getfixtureurl,
post=False,
parameters=OrderedDict([('b', '4'),
('d', '3')]))
assert downloader.get_json()['args'] == OrderedDict([('b', '4'),
('d', '3'),
('id', '10'),
('lala', 'a')
])
downloader.download('%s?id=3&lala=b' % postfixtureurl,
post=True,
parameters=OrderedDict([('a', '3'),
('c', '2')]))
assert downloader.get_json()['form'] == OrderedDict([('a', '3'),
('c', '2'),
('id', '3'),
('lala', 'b')
])
def test_download(self, fixtureurl, fixturenotexistsurl):
with pytest.raises(DownloadError), Download() as download:
download.download('NOTEXIST://NOTEXIST.csv')
with pytest.raises(DownloadError), Download() as download:
download.download(fixturenotexistsurl)
with Download() as download:
result = download.download(fixtureurl)
assert result.headers['Content-Length'] == '479'
def test_setup_stream(self, fixtureurl, fixturenotexistsurl):
with pytest.raises(DownloadError), Download() as download:
download.setup_stream('NOTEXIST://NOTEXIST.csv')
with pytest.raises(DownloadError), Download() as download:
download.setup_stream(fixturenotexistsurl)
with Download() as download:
download.setup_stream(fixtureurl)
headers = download.response.headers
assert headers['Content-Length'] == '479'
def test_download_file(self, fixtureurl, fixturenotexistsurl):
tmpdir = tempfile.gettempdir()
with pytest.raises(DownloadError), Download() as download:
download.download_file('NOTEXIST://NOTEXIST.csv', tmpdir)
with pytest.raises(DownloadError), Download() as download:
download.download_file(fixturenotexistsurl)
with Download() as download:
f = download.download_file(fixtureurl, tmpdir)
fpath = abspath(f)
unlink(f)
assert fpath == abspath(join(tmpdir, 'test_data.csv'))
def test_init(self, downloaderfolder):
basicauthfile = join(downloaderfolder, 'basicauth.txt')
with Download(basicauthfile=basicauthfile) as download:
assert download.session.auth == ('testuser', 'testpass')
with pytest.raises(DownloadError):
Download(auth=('u', 'p'), basicauth='Basic xxxxxxxxxxxxxxxx')
with pytest.raises(DownloadError):
Download(auth=('u', 'p'), basicauthfile=join('lala', 'lala.txt'))
with pytest.raises(DownloadError):
Download(basicauth='Basic xxxxxxxxxxxxxxxx',
basicauthfile=join('lala', 'lala.txt'))
with pytest.raises(IOError):
Download(basicauthfile='NOTEXIST')
def test_download_file(self, tmpdir, fixtureurl, fixturenotexistsurl,
getfixtureurl, postfixtureurl):
tmpdir = str(tmpdir)
with pytest.raises(DownloadError), Download() as downloader:
downloader.download_file('NOTEXIST://NOTEXIST.csv', tmpdir)
with pytest.raises(DownloadError), Download() as downloader:
downloader.download_file(fixturenotexistsurl)
with Download() as downloader:
f = downloader.download_file(fixtureurl, folder=tmpdir)
fpath = abspath(f)
remove(f)
assert fpath == abspath(join(tmpdir, 'test_data.csv'))
filename = 'myfilename.txt'
f = downloader.download_file(fixtureurl,
folder=tmpdir,
filename=filename)
fpath = abspath(f)
remove(f)
assert fpath == abspath(join(tmpdir, filename))
f = downloader.download_file('%s?id=10&lala=a' % getfixtureurl,
post=False,
parameters=OrderedDict([('b', '4'),
('d', '3')]),
folder=tmpdir,
filename=filename)
fpath = abspath(f)
with open(fpath, 'rt') as fi:
text = fi.read()
assert '"id": "10"' in text
assert '"lala": "a"' in text
assert '"b": "4"' in text
assert '"d": "3"' in text
remove(f)
assert fpath == abspath(join(tmpdir, filename))
f = downloader.download_file('%s?id=3&lala=b' % postfixtureurl,
post=True,
parameters=OrderedDict([('a', '3'),
('c', '2')]),
folder=tmpdir,
filename=filename)
fpath = abspath(f)
with open(fpath, 'rt') as fi:
text = fi.read()
assert '"id": "3"' in text
assert '"lala": "b"' in text
assert '"a": "3"' in text
assert '"c": "2"' in text
remove(f)
assert fpath == abspath(join(tmpdir, filename))
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
configuration = Configuration.read()
countries_path = join('config', configuration['countries_filename'])
indicators_url = configuration['indicators_url']
mvam_url = configuration['mvam_url']
showcase_url = configuration['showcase_url']
countries = get_countries(countries_path, downloader)
variables = get_mvamvariables(indicators_url, downloader)
logger.info('Number of datasets to upload: %d' % len(countries))
for info, country in progress_storing_tempdir('WFPFoodSecurity',
countries, 'iso3'):
dataset, showcase, bites_disabled = \
generate_dataset_and_showcase(mvam_url, showcase_url, downloader, info['folder'],
country, variables)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(bites_disabled=bites_disabled)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: WFP Food Security',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def test_download_tabular_rows_as_dicts(self, fixtureprocessurl):
with Download() as downloader:
result = downloader.download_tabular_rows_as_dicts(
fixtureprocessurl, headers=2)
self.fix_strings(result)
assert result == {
"coal": {
"header2": "3",
"header3": "7.4",
"header4": "needed",
},
"gas": {
"header2": "2",
"header3": "6.5",
"header4": "n/a"
},
}
result = downloader.download_tabular_rows_as_dicts(
fixtureprocessurl, headers=2, keycolumn=2)
self.fix_strings(result)
assert result == {
"2": {
"header1": "gas",
"header3": "6.5",
"header4": "n/a"
},
"3": {
"header1": "coal",
"header3": "7.4",
"header4": "needed",
},
}
def data(self, configuration):
resources = configuration["resources"]
download_url = (Path(__file__).resolve().parent / "fixtures").as_uri()
print(download_url)
return get_countriesdata(download_url, resources,
Download(user_agent="test"))
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
download_url = configuration["download_url"]
with Download() as downloader:
countries, headers, countriesdata = get_countriesdata(download_url, downloader)
logger.info(f"Number of countries: {len(countriesdata)}")
for info, country in progress_storing_tempdir("UCDP", countries, "iso3"):
folder = info["folder"]
dataset, showcase = generate_dataset_and_showcase(
folder, country, countriesdata[country["iso3"]], headers
)
if dataset:
dataset.update_from_yaml()
dataset["notes"] = dataset["notes"].replace(
"\n", " \n"
) # ensure markdown has line breaks
dataset.generate_resource_view(1)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UCDP",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main():
"""Generate dataset and create it in HDX"""
filelist_url = Configuration.read()['filelist_url']
country_group_url = Configuration.read()['country_group_url']
dataset_codes = Configuration.read()['dataset_codes']
showcase_base_url = Configuration.read()['showcase_base_url']
with temp_dir('faostat') as folder:
with Download() as downloader:
indicatortypes = get_indicatortypesdata(filelist_url, downloader)
countriesdata = get_countriesdata(country_group_url, downloader)
logger.info('Number of indicator types to upload: %d' %
len(dataset_codes))
for dataset_code in dataset_codes:
datasets, showcases = generate_datasets_and_showcases(
downloader, folder, dataset_codes[dataset_code],
indicatortypes[dataset_code], countriesdata,
showcase_base_url)
logger.info('Number of datasets to upload: %d' % len(datasets))
for i, dataset in enumerate(datasets):
logger.info('Creating dataset: %s' % dataset['title'])
dataset.preview_off()
dataset.create_in_hdx()
showcase = showcases[i]
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
base_url = configuration['base_url']
with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'),
extra_params_lookup=lookup) as downloader:
downloader.session.mount(
'http://',
HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100))
downloader.session.mount(
'https://',
HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100))
countries = get_countries(base_url, downloader)
logger.info('Number of countries: %d' % len(countries))
for folder, country in progress_storing_tempdir(
'DHS', countries, 'iso3'):
tags = get_tags(base_url, downloader, country['dhscode'])
dataset, subdataset, showcase, bites_disabled = \
generate_datasets_and_showcase(configuration, base_url, downloader, folder, country, tags)
if dataset:
createdataset(dataset)
resource_view = generate_resource_view(
dataset, bites_disabled=bites_disabled['national'])
resource_view.create_in_hdx()
showcase.create_in_hdx()
showcase.add_dataset(dataset)
if subdataset:
createdataset(subdataset)
showcase.add_dataset(subdataset)
subdataset.generate_resource_view(
bites_disabled=bites_disabled['subnational'])
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
config = Configuration.read()
project_config = {
key: value
for key, value in config.items() if key.startswith("CV")
}
qc_indicators = config.get("qc_indicators", {})
countries, countriesdata, headers = get_all_countriesdata(
project_config, downloader)
logger.info("Number of datasets to upload: %d" % len(countries))
for info, country in progress_storing_tempdir("UNICEFSAM", countries,
"iso3"):
dataset, showcase, bites_disabled = generate_dataset_and_showcase(
info["folder"], country, countriesdata[country["iso3"]],
headers, project_config, qc_indicators)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(1,
bites_disabled=bites_disabled,
indicators=qc_indicators)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UNICEF Sam",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main(
output_dir,
saved_dir,
save,
use_saved,
dportal_params,
whattorun,
filterdate,
**ignore,
):
logger.info(f"##### hdx-scraper-iati-viz version {VERSION:.1f} ####")
configuration = Configuration.read()
output_dir = f"{output_dir}_{whattorun}"
rmtree(output_dir, ignore_errors=True)
mkdir(output_dir)
with Download() as downloader:
retriever = Retrieve(
downloader,
configuration["fallback_dir"],
f"{saved_dir}_{whattorun}",
output_dir,
save,
use_saved,
)
today = datetime.utcnow().isoformat()
start(
configuration,
today,
retriever,
output_dir,
dportal_params,
whattorun,
filterdate,
)
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers,
basic_auths, nojson, **ignore):
logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION)
configuration = Configuration.read()
with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader:
if scrapers:
logger.info('Updating only scrapers: %s' % scrapers)
tabs = configuration['tabs']
if updatetabs is None:
updatetabs = list(tabs.keys())
logger.info('Updating all tabs')
else:
logger.info('Updating only these tabs: %s' % updatetabs)
noout = nooutput(updatetabs)
if excel_path:
excelout = exceloutput(excel_path, tabs, updatetabs)
else:
excelout = noout
if gsheet_auth:
gsheets = googlesheets(configuration, gsheet_auth, updatesheets,
tabs, updatetabs)
else:
gsheets = noout
if nojson:
jsonout = noout
else:
jsonout = jsonoutput(configuration, updatetabs)
outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout}
admininfo = AdminInfo.setup(downloader)
get_indicators(configuration, downloader, admininfo, outputs,
updatetabs, scrapers, basic_auths)
excelout.save()
jsonout.add_additional_json(downloader)
jsonout.save(hrp_iso3s=admininfo.hrp_iso3s)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
hdro_url = configuration['hdro_url']
qc_indicators = configuration['qc_indicators']
with Download() as downloader:
countriesdata = get_countriesdata(hdro_url, downloader)
countries = [{
'iso3': countryiso
} for countryiso in sorted(countriesdata.keys())]
logger.info('Number of countries to upload: %d' % len(countries))
for info, country in progress_storing_tempdir('HDRO', countries,
'iso3'):
countryiso = country['iso3']
countrydata = countriesdata[countryiso]
dataset, showcase, bites_disabled = generate_dataset_and_showcase(
info['folder'], countryiso, countrydata, qc_indicators)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(-1,
bites_disabled=bites_disabled,
indicators=qc_indicators)
dataset.create_in_hdx(remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: HDRO',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def test_run(self, configuration, fixtures_dir):
with temp_dir("TestIATIViz",
delete_on_success=True,
delete_on_failure=False) as tempdir:
with Download(user_agent="test") as downloader:
retriever = Retrieve(
downloader,
tempdir,
fixtures_dir,
tempdir,
save=False,
use_saved=True,
)
today = "2021-05-06"
start(
configuration,
today,
retriever,
tempdir,
dportal_params=None,
whattorun="covid",
filterdate="2020-01",
)
for filename in ("flows", "transactions", "reporting_orgs"):
csv_filename = f"{filename}.csv"
expected_file = join(fixtures_dir, csv_filename)
actual_file = join(tempdir, csv_filename)
assert_files_same(expected_file, actual_file)
json_filename = f"{filename}.json"
expected_file = join(fixtures_dir, json_filename)
actual_file = join(tempdir, json_filename)
assert filecmp.cmp(expected_file, actual_file)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
resources = configuration["resources"]
fields = configuration["fields"]
source_directory = configuration["source_directory"]
download_url = Path(source_directory).resolve().as_uri()
with Download() as downloader:
countries, headers, countriesdata = get_countriesdata(
download_url, resources, downloader)
logger.info("Number of countries: %d" % len(countriesdata))
for info, country in progress_storing_tempdir("UNHCR_population",
countries, "iso3"):
folder = info["folder"]
dataset, showcase = generate_dataset_and_showcase(
folder, country, countriesdata[country["iso3"]], headers,
resources, fields)
if dataset:
dataset.update_from_yaml()
dataset["notes"] = dataset["notes"].replace(
"\n", " \n") # ensure markdown has line breaks
dataset.generate_resource_view(1)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UNHCR population",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
indicators = configuration["indicators"]
json_url = configuration["json_url"]
with Download() as downloader:
indicators_metadata = get_indicators_metadata(json_url, downloader,
indicators)
countriesdata, countries = get_countriesdata(json_url, downloader,
indicators)
logger.info(f"Number of countries to upload: {len(countries)}")
for info, country in progress_storing_tempdir("WorldPop", countries,
"iso3"):
countryiso = country["iso3"]
datasets, showcases = generate_datasets_and_showcases(
downloader, countryiso, indicators_metadata,
countriesdata[countryiso])
for dataset in datasets:
dataset.update_from_yaml()
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: WorldPop",
batch=info["batch"],
)
for showcase in showcases.get(dataset["name"], list()):
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def get_soup(
url: str,
downloader: Download = None,
user_agent: Optional[str] = None,
user_agent_config_yaml: Optional[str] = None,
user_agent_lookup: Optional[str] = None,
**kwargs: Any,
) -> BeautifulSoup:
"""
Get BeautifulSoup object for a url. Requires either global user agent to be set or appropriate user agent
parameter(s) to be completed.
Args:
url (str): url to read
downloader (Download): Download object. Defaults to creating a Download object with given user agent values.
user_agent (Optional[str]): User agent string. HDXPythonUtilities/X.X.X- is prefixed.
user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied.
Returns:
BeautifulSoup: The BeautifulSoup object for a url
"""
if not downloader:
downloader = Download(user_agent, user_agent_config_yaml,
user_agent_lookup, **kwargs)
response = downloader.download(url)
return BeautifulSoup(response.text, "html.parser")
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers, basic_auths, other_auths, nojson,
countries_override, save, use_saved, **ignore):
logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION)
configuration = Configuration.read()
with temp_dir() as temp_folder:
with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader:
retriever = Retrieve(downloader, temp_folder, 'saved_data', temp_folder, save, use_saved)
if scrapers:
logger.info('Updating only scrapers: %s' % scrapers)
tabs = configuration['tabs']
if updatetabs is None:
updatetabs = list(tabs.keys())
logger.info('Updating all tabs')
else:
logger.info('Updating only these tabs: %s' % updatetabs)
noout = NoOutput(updatetabs)
if excel_path:
excelout = ExcelOutput(excel_path, tabs, updatetabs)
else:
excelout = noout
if gsheet_auth:
gsheets = GoogleSheets(configuration, gsheet_auth, updatesheets, tabs, updatetabs)
else:
gsheets = noout
if nojson:
jsonout = noout
else:
jsonout = JsonOutput(configuration, updatetabs)
outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout}
today = datetime.now()
countries_to_save = get_indicators(configuration, today, retriever, outputs, updatetabs, scrapers,
basic_auths, other_auths, countries_override)
jsonout.add_additional_json(downloader, today=today)
jsonout.save(countries_to_save=countries_to_save)
excelout.save()
def test_download_tabular_rows_as_dicts(self, fixtureprocessurl):
with Download() as downloader:
result = downloader.download_tabular_rows_as_dicts(
fixtureprocessurl, headers=2)
self.fix_strings(result)
assert result == {
'coal': {
'header2': '3',
'header3': '7.4',
'header4': 'needed'
},
'gas': {
'header2': '2',
'header3': '6.5',
'header4': 'n/a'
}
}
result = downloader.download_tabular_rows_as_dicts(
fixtureprocessurl, headers=2, keycolumn=2)
self.fix_strings(result)
assert result == {
'2': {
'header1': 'gas',
'header3': '6.5',
'header4': 'n/a'
},
'3': {
'header1': 'coal',
'header3': '7.4',
'header4': 'needed'
}
}
def test_get_tabular_rows_as_list(self, fixtureprocessurl):
with Download() as downloader:
rows = list(downloader.get_tabular_rows_as_list(fixtureprocessurl))
assert rows == [
["la1", "ha1", "ba1", "ma1"],
["header1", "header2", "header3", "header4"],
["coal", "3", "7.4", "needed"],
["gas", "2", "6.5", "n/a"],
]
def test_get_indicators(self, configuration, folder):
with temp_dir('TestCovidViz') as tempdir:
with Download(user_agent='test') as downloader:
tabs = configuration['tabs']
noout = nooutput(tabs)
jsonout = jsonoutput(configuration, tabs)
outputs = {'gsheets': noout, 'excel': noout, 'json': jsonout}
get_indicators(configuration, downloader, outputs, tabs, scrapers=['ifi', 'who', 'covid_trend'])
filepath = jsonout.save(tempdir)
assert_files_same(filepath, join(folder, 'test_tabular.json'))
def test_download_tabular_key_value(self, fixtureurl, fixtureprocessurl):
with Download() as downloader:
result = downloader.download_tabular_key_value(fixtureurl,
file_type='csv')
assert result == {'615': '2231RTA', 'GWNO': 'EVENT_ID_CNTY'}
result = downloader.download_tabular_key_value(fixtureprocessurl,
headers=2)
assert result == {'coal': '3', 'gas': '2'}
with pytest.raises(DownloadError):
downloader.download_tabular_key_value(
'NOTEXIST://NOTEXIST.csv')
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
indicators = Configuration.read()['indicators']
tags = Configuration.read()['tags']
folder = get_temp_dir('IDMC')
datasets, showcase, headersdata, countriesdata = generate_indicator_datasets_and_showcase(
downloader, folder, indicators, tags)
showcase_not_added = True
countries = [{'iso3': x} for x in sorted(countriesdata)]
logger.info('Number of indicator datasets to upload: %d' %
len(indicators))
logger.info('Number of country datasets to upload: %d' %
len(countries))
for i, info, nextdict in multiple_progress_storing_tempdir(
'IDMC', [indicators, countries], ['name', 'iso3']):
folder = info['folder']
batch = info['batch']
if i == 0:
if showcase_not_added:
showcase.create_in_hdx()
showcase_not_added = False
dataset = datasets[nextdict['name']]
dataset.update_from_yaml()
dataset.generate_resource_view(
join('config', nextdict['resourceview']))
dataset.create_in_hdx(remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: IDMC',
batch=batch)
showcase.add_dataset(dataset)
else:
countryiso = nextdict['iso3']
countrydata = countriesdata[countryiso]
dataset, showcase, bites_disabled = \
generate_country_dataset_and_showcase(downloader, folder, headersdata, countryiso, countrydata, datasets, tags)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(
bites_disabled=bites_disabled)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: IDMC',
batch=batch)
resources = dataset.get_resources()
resource_ids = [
x['id'] for x in sorted(resources,
key=lambda x: len(x['name']),
reverse=True)
]
dataset.reorder_resources(resource_ids, hxl_update=False)
def test_download_tabular_key_value(self, fixtureurl, fixtureprocessurl):
with Download() as downloader:
result = downloader.download_tabular_key_value(fixtureurl,
file_type="csv")
assert result == {"615": "2231RTA", "GWNO": "EVENT_ID_CNTY"}
result = downloader.download_tabular_key_value(fixtureprocessurl,
headers=2)
assert result == {"coal": "3", "gas": "2"}
with pytest.raises(DownloadError):
downloader.download_tabular_key_value(
"NOTEXIST://NOTEXIST.csv")