def test_run(self, configuration, fixtures_dir):
with temp_dir("TestIATIViz",
delete_on_success=True,
delete_on_failure=False) as tempdir:
with Download(user_agent="test") as downloader:
retriever = Retrieve(
downloader,
tempdir,
fixtures_dir,
tempdir,
save=False,
use_saved=True,
)
today = "2021-05-06"
start(
configuration,
today,
retriever,
tempdir,
dportal_params=None,
whattorun="covid",
filterdate="2020-01",
)
for filename in ("flows", "transactions", "reporting_orgs"):
csv_filename = f"{filename}.csv"
expected_file = join(fixtures_dir, csv_filename)
actual_file = join(tempdir, csv_filename)
assert_files_same(expected_file, actual_file)
json_filename = f"{filename}.json"
expected_file = join(fixtures_dir, json_filename)
actual_file = join(tempdir, json_filename)
assert filecmp.cmp(expected_file, actual_file)
def test_compare_files(self, testfile1, testfile2):
result = compare_files(testfile1, testfile2)
assert result == [
"- coal ,3 ,7.4 ,'needed'\n", '? ^\n',
"+ coal ,1 ,7.4 ,'notneeded'\n",
'? ^ +++\n'
]
assert_files_same(testfile1, testfile1)
def test_fts(self, folder, downloaderfts, lc_country_list):
resource_updates = dict()
filename = 'Lake_Chad_Basin_Appeal_Status.csv'
expected = join('tests', 'fixtures', filename)
actual = join(folder, filename)
resource_updates['fts'] = {'path': actual}
update_fts('http://lala/', downloaderfts, lc_country_list,
resource_updates)
assert_files_same(expected, actual)
def test_schemacreator(self):
start_url = 'https://raw.githubusercontent.com/OCHA-DAP/tools-datacheck-validation/master/tests/fixtures/COD_External.json'
base_url = 'https://raw.githubusercontent.com/OCHA-DAP/tools-datacheck-validation/master/tests/fixtures/%s_layers.json'
template_path = 'validation-schema-pcodes.json'
output_folder = gettempdir()
schemacreator(start_url, base_url, template_path, output_folder)
expected = join('tests', 'fixtures',
'validation-schema-pcodes-syr.json')
result = join(output_folder, 'validation-schema-pcodes-syr.json')
assert_files_same(expected, result)
def test_cbpf(self, folder, today, downloadercbpf, valid_ssd_adm1_names,
replace_ssd_values):
resource_updates = dict()
filename = 'South_Sudan_Country_Based_Pool_Funds.csv'
expected = join('tests', 'fixtures', filename)
actual = join(folder, filename)
resource_updates['cbpf'] = {'path': actual}
update_cbpf('http://mama/', downloadercbpf, 'SSD19', today,
valid_ssd_adm1_names, replace_ssd_values, resource_updates)
assert_files_same(expected, actual)
def test_get_indicators(self, configuration, folder):
with temp_dir('TestCovidViz') as tempdir:
with Download(user_agent='test') as downloader:
tabs = configuration['tabs']
noout = nooutput(tabs)
jsonout = jsonoutput(configuration, tabs)
outputs = {'gsheets': noout, 'excel': noout, 'json': jsonout}
get_indicators(configuration, downloader, outputs, tabs, scrapers=['ifi', 'who', 'covid_trend'])
filepath = jsonout.save(tempdir)
assert_files_same(filepath, join(folder, 'test_tabular.json'))
def test_save_json(self, tmpdir, saverfolder, filename, pretty, sortkeys):
test_path = join(str(tmpdir), filename)
ref_path = join(saverfolder, filename)
save_json(
TestLoader.json_to_write,
test_path,
pretty=pretty,
sortkeys=sortkeys,
)
assert_files_same(ref_path, test_path)
dct = json.loads(json.dumps(TestLoader.json_to_write))
save_json(dct, test_path, pretty=pretty, sortkeys=sortkeys)
assert_files_same(ref_path, test_path)
def test_generate_dataset_and_showcase(self, configuration, data):
with temp_dir("ucdp") as folder:
countries, headers, countriesdata = data
dataset, showcase = generate_dataset_and_showcase(
folder, countries[1], countriesdata["BGD"], headers)
assert dataset == TestUCDP.dataset
resources = dataset.get_resources()
assert resources == TestUCDP.resources
file = "conflict_data_BGD.csv"
assert_files_same(join("tests", "fixtures", file),
join(folder, file))
file = "qc_conflict_data_BGD.csv"
assert_files_same(join("tests", "fixtures", file),
join(folder, file))
assert showcase == {
"name":
"ucdp-data-for-bangladesh-showcase",
"title":
"Bangladesh - Data on Conflict Events",
"notes":
"Conflict Data Dashboard for Bangladesh",
"url":
"https://ucdp.uu.se/#country/771",
"image_url":
"https://pbs.twimg.com/profile_images/832251660718178304/y-LWa5iK_200x200.jpg",
"tags": [
{
"name": "hxl",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "violence and conflict",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "protests",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "security incidents",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
],
}
def test_generate_dataset_and_showcase(self, configuration, data):
with temp_dir('ucdp') as folder:
countries, headers, countriesdata = data
dataset, showcase = generate_dataset_and_showcase(
folder, countries[1], countriesdata['BGD'], headers)
assert dataset == TestUCDP.dataset
resources = dataset.get_resources()
assert resources == TestUCDP.resources
file = 'conflict_data_BGD.csv'
assert_files_same(join('tests', 'fixtures', file),
join(folder, file))
file = 'qc_conflict_data_BGD.csv'
assert_files_same(join('tests', 'fixtures', file),
join(folder, file))
assert showcase == {
'name':
'ucdp-data-for-bangladesh-showcase',
'title':
'Bangladesh - Conflict Data',
'notes':
'Conflict Data Dashboard for Bangladesh',
'url':
'https://ucdp.uu.se/#country/771',
'image_url':
'https://pbs.twimg.com/profile_images/832251660718178304/y-LWa5iK_200x200.jpg',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'protests',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'security incidents',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}]
}
def test_ssd_acled(self, folder, today, ssd_country_list,
valid_ssd_adm2_names, replace_ssd_values):
resource_updates = dict()
filename = 'South_Sudan_Recent_Conflict_Events.csv'
expected_events = join('tests', 'fixtures', filename)
actual_events = join(folder, filename)
resource_updates['acled_events'] = {'path': actual_events}
filename = 'South_Sudan_Recent_Conflict_Event_Total_Fatalities.csv'
expected_fatalities = join('tests', 'fixtures', filename)
actual_fatalities = join(folder, filename)
resource_updates['acled_fatalities'] = {'path': actual_fatalities}
update_ssd_acled(
today,
'https://raw.githubusercontent.com/mcarans/hdxscraper-mapexplorer/master/tests/fixtures/ACLEDSouthSudan.csv?',
ssd_country_list, valid_ssd_adm2_names, replace_ssd_values,
resource_updates)
assert_files_same(expected_events, actual_events)
assert_files_same(expected_fatalities, actual_fatalities)
def test_get_indicators(self, configuration, folder):
configuration = Configuration.read()
countries = configuration['countries']
palestine_country_code, _ = Country.get_iso3_country_code_fuzzy(
'Palestine')
df_indicators, df_timeseries, df_cumulative = get_indicators(
folder, countries, palestine_country_code, True)
with temp_dir('PA-COVID-TEST') as outdir:
ifilename = 'indicators.csv'
ioutputfile = join(outdir, ifilename)
df_indicators.to_csv(ioutputfile)
tfilename = 'timeseries.csv'
toutputfile = join(outdir, tfilename)
df_timeseries.to_csv(toutputfile)
cfilename = 'cumulative.csv'
coutputfile = join(outdir, cfilename)
df_cumulative.to_csv(coutputfile)
assert_files_same(join(folder, ifilename), ioutputfile)
assert_files_same(join(folder, tfilename), toutputfile)
assert_files_same(join(folder, cfilename), coutputfile)
def test_generate_datasets_and_showcase(self, configuration, downloader):
with temp_dir('DHS') as folder:
dataset, subdataset, showcase, bites_disabled = \
generate_datasets_and_showcase(configuration, 'http://haha/', downloader, folder, TestDHS.country, TestDHS.tags)
assert dataset == TestDHS.dataset
resources = dataset.get_resources()
assert resources == TestDHS.resources
assert subdataset == TestDHS.subdataset
assert subdataset.get_resources() == TestDHS.subresources
assert showcase == {'name': 'dhs-data-for-afghanistan-showcase', 'title': 'Final Report', 'notes': 'Afghanistan Demographic and Health Survey 2015',
'url': 'https://www.dhsprogram.com/pubs/pdf/FR323/FR323.pdf', 'image_url': 'https://www.dhsprogram.com/publications/images/thumbnails/FR323.jpg',
'tags': [{'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}, {'name': 'health', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}, {'name': 'demographics', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}]}
assert bites_disabled == {'national': [False, False, False], 'subnational': [False, False, False]}
file = 'DHS Quickstats_national_AFG.csv'
assert_files_same(join('tests', 'fixtures', file), join(folder, file))
file = 'DHS Mobile_national_AFG.csv'
assert_files_same(join('tests', 'fixtures', file), join(folder, file))
file = 'DHS Quickstats_subnational_AFG.csv'
assert_files_same(join('tests', 'fixtures', file), join(folder, file))
def test_generate_dataset_and_showcase(self, configuration, downloader):
with temp_dir("faostat-test") as folder:
filelist_url = configuration["filelist_url"]
showcase_base_url = configuration["showcase_base_url"]
(
dataset,
showcase,
bites_disabled,
qc_indicators,
) = generate_dataset_and_showcase(
"Food Security",
TestFaostat.indicatorsets,
TestFaostat.country,
TestFaostat.countrymapping,
showcase_base_url,
filelist_url,
downloader,
folder,
)
assert dataset == {
"name":
"faostat-food-security-indicators-for-afghanistan",
"title":
"Afghanistan - Food Security Indicators",
"notes":
"Food Security Indicators for Afghanistan.\n\nContains data from the FAOSTAT [bulk data service](http://lala/datasets_E.json).",
"maintainer":
"196196be-6037-4488-8b71-d786adf4c081",
"owner_org":
"ed727a5b-3e6e-4cd6-b97e-4a71532085e6",
"data_update_frequency":
"365",
"subnational":
"0",
"tags": [
{
"name": "hxl",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "indicators",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "food security",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
],
"groups": [{
"name": "afg"
}],
"dataset_date":
"[1999-01-01T00:00:00 TO 2014-12-31T00:00:00]",
}
resources = dataset.get_resources()
assert resources == [
{
"name":
"Suite of Food Security Indicators for Afghanistan",
"description":
"*Suite of Food Security Indicators:*\nFor detailed description of the indicators below see attached document: Average Dietary Supply Adequacy;...",
"format": "csv",
"resource_type": "file.upload",
"url_type": "upload",
},
{
"name":
"QuickCharts-Suite of Food Security Indicators for Afghanistan",
"description": "Cut down data for QuickCharts",
"format": "csv",
"resource_type": "file.upload",
"url_type": "upload",
},
]
assert showcase == {
"name":
"faostat-food-security-indicators-for-afghanistan-showcase",
"title":
"Afghanistan - Food Security Indicators",
"notes":
"Food Security Data Dashboard for Afghanistan",
"url":
"http://www.fao.org/faostat/en/#country/AFG",
"image_url":
"https://pbs.twimg.com/profile_images/1375385494167691269/Bc49-Yx8_400x400.jpg",
"tags": [
{
"name": "hxl",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "indicators",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
{
"name": "food security",
"vocabulary_id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
},
],
}
assert bites_disabled == [False, True, True]
assert qc_indicators == [
{
"code": "21010",
"title": "Average dietary energy supply adequacy",
"unit": "Percentage",
},
{
"code": "210041",
"title": "Prevalence of undernourishment",
"unit": "Percentage",
},
{
"code": "21034",
"title":
"Percentage of arable land equipped for irrigation",
"unit": "Percentage",
},
]
file = "Suite of Food Security Indicators_AFG.csv"
assert_files_same(join("tests", "fixtures", file),
join(folder, file))
file = f"qc_{file}"
assert_files_same(join("tests", "fixtures", file),
join(folder, file))
def test_generate_datasets_and_showcase(self, configuration, downloader):
with temp_dir('idmc') as folder:
# indicator dataset test
indicators = Configuration.read()['indicators']
tags = Configuration.read()['tags']
datasets, showcase, headersdata, countriesdata = generate_indicator_datasets_and_showcase(
downloader, folder, indicators, tags)
assert datasets == {
'displacement_data': {
'name':
'idmc-internally-displaced-persons-idps',
'title':
'Internally displaced persons - IDPs',
'maintainer':
'196196be-6037-4488-8b71-d786adf4c081',
'owner_org':
'647d9d8c-4cac-4c33-b639-649aad1c2893',
'data_update_frequency':
'365',
'subnational':
'0',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}],
'notes':
"Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).",
'methodology_other':
'Methodology',
'caveats':
'Caveats',
'groups': [{
'name': 'world'
}],
'dataset_date':
'01/01/2008-12/31/2018'
},
'disaster_data': {
'name':
'idmc-internally-displaced-persons-idps-new-displacement-associated-with-disasters',
'title':
'Internally displaced persons - IDPs (new displacement associated with disasters)',
'maintainer':
'196196be-6037-4488-8b71-d786adf4c081',
'owner_org':
'647d9d8c-4cac-4c33-b639-649aad1c2893',
'data_update_frequency':
'365',
'subnational':
'0',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}],
'notes':
"Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).",
'methodology_other':
'Methodology',
'caveats':
'Caveats',
'groups': [{
'name': 'world'
}],
'dataset_date':
'01/01/2008-12/31/2018'
}
}
resources = datasets['displacement_data'].get_resources()
assert resources == [{
'description': 'Internally displaced persons - IDPs',
'format': 'csv',
'name': 'displacement_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}]
resource_name = '%s.%s' % (resources[0]['name'],
resources[0]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
resources = datasets['disaster_data'].get_resources()
resource_name = '%s.%s' % (resources[0]['name'],
resources[0]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
assert resources == [{
'description':
'Internally displaced persons - IDPs (new displacement associated with disasters)',
'format': 'csv',
'name': 'disaster_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}]
assert showcase == {
'image_url':
'http://www.internal-displacement.org/global-report/grid2018/img/ogimage.jpg',
'name':
'idmc-global-report-on-internal-displacement',
'notes':
'Click the image on the right to go to the IDMC Global Report on Internal Displacement',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}],
'title':
'IDMC Global Report on Internal Displacement',
'url':
'http://www.internal-displacement.org/global-report/grid2018/'
}
# country datasets tests
dataset, showcase, disables_bites = generate_country_dataset_and_showcase(
downloader, folder, headersdata, 'AFG', countriesdata['AFG'],
datasets, tags)
assert dataset == TestIDMC.afg_dataset
resources = dataset.get_resources()
assert resources == [{
'description':
'Internally displaced persons - IDPs for Afghanistan',
'format': 'csv',
'name': 'displacement_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}, {
'description':
'Internally displaced persons - IDPs (new displacement associated with disasters) for Afghanistan',
'format': 'csv',
'name': 'disaster_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}]
resource_name = '%s.%s' % (resources[0]['name'],
resources[0]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
resource_name = '%s.%s' % (resources[1]['name'],
resources[1]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
assert showcase == {
'name':
'idmc-idp-data-for-afghanistan-showcase',
'title':
'IDMC Afghanistan Summary Page',
'notes':
'Click the image on the right to go to the IDMC summary page for the Afghanistan dataset',
'url':
'http://www.internal-displacement.org/countries/Afghanistan/',
'image_url':
'http://www.internal-displacement.org/sites/default/files/logo_0.png',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}]
}
assert disables_bites == [False, False, False]
dataset, showcase, disables_bites = generate_country_dataset_and_showcase(
downloader, folder, headersdata, 'TZA', countriesdata['TZA'],
datasets, tags)
assert dataset == {
'name':
'idmc-idp-data-for-united-republic-of-tanzania',
'title':
'United Republic of Tanzania - Internally displaced persons - IDPs',
'maintainer':
'196196be-6037-4488-8b71-d786adf4c081',
'owner_org':
'647d9d8c-4cac-4c33-b639-649aad1c2893',
'data_update_frequency':
'365',
'subnational':
'0',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}],
'groups': [{
'name': 'tza'
}],
'notes':
"Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).",
'methodology_other':
'',
'caveats':
'',
'dataset_date':
'01/01/2011-12/31/2012'
}
resources = dataset.get_resources()
assert resources == [{
'description':
'Internally displaced persons - IDPs for United Republic of Tanzania',
'format': 'csv',
'name': 'displacement_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}, {
'description':
'Internally displaced persons - IDPs (new displacement associated with disasters) for United Republic of Tanzania',
'format': 'csv',
'name': 'disaster_data',
'resource_type': 'file.upload',
'url_type': 'upload'
}]
resource_name = '%s.%s' % (resources[0]['name'],
resources[0]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
resource_name = '%s.%s' % (resources[1]['name'],
resources[1]['format'])
expected_file = join('tests', 'fixtures', resource_name)
actual_file = join(folder, resource_name)
assert_files_same(expected_file, actual_file)
assert showcase == {
'name':
'idmc-idp-data-for-united-republic-of-tanzania-showcase',
'title':
'IDMC United Republic of Tanzania Summary Page',
'notes':
'Click the image on the right to go to the IDMC summary page for the United Republic of Tanzania dataset',
'url':
'http://www.internal-displacement.org/countries/Tanzania/',
'image_url':
'http://www.internal-displacement.org/sites/default/files/logo_0.png',
'tags': [{
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'displacement',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'internally displaced persons - idp',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'violence and conflict',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}]
}
assert disables_bites == [True, True, False]
dataset, showcase, disables_bites = generate_country_dataset_and_showcase(
downloader, folder, headersdata, 'AB9', countriesdata['AB9'],
datasets, tags)
assert dataset is None
assert showcase is None
assert disables_bites is None
def test_generate_dataset_and_showcase(self, configuration):
with temp_dir('HDRO') as folder:
qc_indicators = configuration['qc_indicators']
dataset, showcase, bites_disabled = generate_dataset_and_showcase(
folder, 'AFG', hdro_data, qc_indicators)
assert dataset == {
'name':
'hdro-data-for-afghanistan',
'title':
'Afghanistan - Human Development Indicators',
'maintainer':
'872427e4-7e9b-44d6-8c58-30d5052a00a2',
'owner_org':
'89ebe982-abe9-4748-9dde-cf04632757d6',
'data_update_frequency':
'365',
'subnational':
'0',
'groups': [{
'name': 'afg'
}],
'tags': [{
'name':
'health',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'education',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'socioeconomic',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'demographics',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'development',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'indicators',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}],
'dataset_date':
'[2008-01-01T00:00:00 TO 2019-12-31T00:00:00]'
}
resources = dataset.get_resources()
assert resources == [{
'name': 'Human Development Indicators for Afghanistan',
'description': 'Human development data with HXL tags',
'format': 'csv',
'resource_type': 'file.upload',
'url_type': 'upload'
}, {
'name':
'QuickCharts-Human Development Indicators for Afghanistan',
'description': 'Cut down data for QuickCharts',
'format': 'csv',
'resource_type': 'file.upload',
'url_type': 'upload'
}]
assert showcase == {
'name':
'hdro-data-for-afghanistan-showcase',
'title':
'Indicators for Afghanistan',
'notes':
'Human Development indicators for Afghanistan',
'url':
'http://hdr.undp.org/en/countries/profiles/AFG',
'image_url':
'https://s1.stabroeknews.com/images/2019/12/undp.jpg',
'tags': [{
'name':
'health',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'education',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'socioeconomic',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'demographics',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'development',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'indicators',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}, {
'name':
'hxl',
'vocabulary_id':
'4e61d464-4943-4e97-973a-84673c1aaa87'
}]
}
assert bites_disabled == [False, False, True]
file = 'hdro_indicators_AFG.csv'
assert_files_same(join('tests', 'fixtures', file),
join(folder, file))
file = 'qc_%s' % file
assert_files_same(join('tests', 'fixtures', file),
join(folder, file))
def test_run(self, configuration, fixtures_dir, input_dir):
with temp_dir('TestIATIViz',
delete_on_success=True,
delete_on_failure=False) as tempdir:
with Download(user_agent='test') as downloader:
retriever = Retrieve(downloader,
tempdir,
input_dir,
tempdir,
save=False,
use_saved=True)
wfp = WFPFood(configuration, None, retriever)
countries = wfp.get_countries()
assert len(countries) == 291
assert countries[100:102] == [{
'iso3': 'GTM',
'name': 'Guatemala'
}, {
'iso3': 'GUF',
'name': 'French Guiana'
}]
wfp.build_mappings()
dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase(
'COG', tempdir)
assert dataset == {
'name':
'wfp-food-prices-for-congo',
'title':
'Congo - Food Prices',
'maintainer':
'f1921552-8c3e-47e9-9804-579b14a83ee3',
'owner_org':
'3ecac442-7fed-448d-8f78-b385ef6f84e7',
'data_update_frequency':
'7',
'groups': [{
'name': 'cog'
}],
'subnational':
'1',
'tags': [{
'name':
'commodities',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'prices',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'markets',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'hxl',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}],
'dataset_source':
'CARITAS, GOV, Gvt, National Institute Of Statistics (INS), WFP',
'dataset_date':
'[2011-01-15T00:00:00 TO 2020-07-15T00:00:00]'
}
assert showcase == {
'name':
'wfp-food-prices-for-congo-showcase',
'title':
'Congo - Food Prices showcase',
'notes':
'Congo food prices data from World Food Programme displayed through VAM Economic Explorer',
'url':
'http://dataviz.vam.wfp.org/economic_explorer/prices?iso3=COG',
'image_url':
'http://dataviz.vam.wfp.org/_images/home/3_economic.jpg',
'tags': [{
'name':
'commodities',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'prices',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'markets',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}, {
'name':
'hxl',
'vocabulary_id':
'b891512e-9516-4bf5-962a-7a289772a2a1'
}]
}
assert qc_indicators == [{
'code':
'Brazzaville-Brazzaville-Total-Groundnuts (shelled)-KG-XAF',
'title': 'Price of Groundnuts (shelled) in Total',
'unit': 'Currency XAF',
'description':
'Price of Groundnuts (shelled) (XAF/KG) in Brazzaville/Total',
'code_col': '#meta+code',
'value_col': '#value',
'date_col': '#date'
}, {
'code':
'Point-Noire-Pointe-Noire-Grand marché/Fond Ntié-Ntié/Nkouikou-Oil (vegetable)-L-XAF',
'title':
'Price of Oil (vegetable) in Grand marché/Fond Ntié-Ntié/Nkouikou',
'unit': 'Currency XAF',
'description':
'Price of Oil (vegetable) (XAF/L) in Point-Noire/Pointe-Noire/Grand marché/Fond Ntié-Ntié/Nkouikou',
'code_col': '#meta+code',
'value_col': '#value',
'date_col': '#date'
}, {
'code':
'Pool-Kinkala-Kinkala-Rice (mixed, low quality)-KG-XAF',
'title': 'Price of Rice (mixed, low quality) in Kinkala',
'unit': 'Currency XAF',
'description':
'Price of Rice (mixed, low quality) (XAF/KG) in Pool/Kinkala',
'code_col': '#meta+code',
'value_col': '#value',
'date_col': '#date'
}]
for filename in ('wfp_food_prices_cog',
'wfp_food_prices_cog_qc'):
csv_filename = f'{filename}.csv'
expected_file = join(fixtures_dir, csv_filename)
actual_file = join(tempdir, csv_filename)
assert_files_same(expected_file, actual_file)
