def test_run(self, configuration, fixtures_dir): with temp_dir("TestIATIViz", delete_on_success=True, delete_on_failure=False) as tempdir: with Download(user_agent="test") as downloader: retriever = Retrieve( downloader, tempdir, fixtures_dir, tempdir, save=False, use_saved=True, ) today = "2021-05-06" start( configuration, today, retriever, tempdir, dportal_params=None, whattorun="covid", filterdate="2020-01", ) for filename in ("flows", "transactions", "reporting_orgs"): csv_filename = f"{filename}.csv" expected_file = join(fixtures_dir, csv_filename) actual_file = join(tempdir, csv_filename) assert_files_same(expected_file, actual_file) json_filename = f"{filename}.json" expected_file = join(fixtures_dir, json_filename) actual_file = join(tempdir, json_filename) assert filecmp.cmp(expected_file, actual_file)
def test_compare_files(self, testfile1, testfile2): result = compare_files(testfile1, testfile2) assert result == [ "- coal ,3 ,7.4 ,'needed'\n", '? ^\n', "+ coal ,1 ,7.4 ,'notneeded'\n", '? ^ +++\n' ] assert_files_same(testfile1, testfile1)
def test_fts(self, folder, downloaderfts, lc_country_list): resource_updates = dict() filename = 'Lake_Chad_Basin_Appeal_Status.csv' expected = join('tests', 'fixtures', filename) actual = join(folder, filename) resource_updates['fts'] = {'path': actual} update_fts('http://lala/', downloaderfts, lc_country_list, resource_updates) assert_files_same(expected, actual)
def test_schemacreator(self): start_url = 'https://raw.githubusercontent.com/OCHA-DAP/tools-datacheck-validation/master/tests/fixtures/COD_External.json' base_url = 'https://raw.githubusercontent.com/OCHA-DAP/tools-datacheck-validation/master/tests/fixtures/%s_layers.json' template_path = 'validation-schema-pcodes.json' output_folder = gettempdir() schemacreator(start_url, base_url, template_path, output_folder) expected = join('tests', 'fixtures', 'validation-schema-pcodes-syr.json') result = join(output_folder, 'validation-schema-pcodes-syr.json') assert_files_same(expected, result)
def test_cbpf(self, folder, today, downloadercbpf, valid_ssd_adm1_names, replace_ssd_values): resource_updates = dict() filename = 'South_Sudan_Country_Based_Pool_Funds.csv' expected = join('tests', 'fixtures', filename) actual = join(folder, filename) resource_updates['cbpf'] = {'path': actual} update_cbpf('http://mama/', downloadercbpf, 'SSD19', today, valid_ssd_adm1_names, replace_ssd_values, resource_updates) assert_files_same(expected, actual)
def test_get_indicators(self, configuration, folder): with temp_dir('TestCovidViz') as tempdir: with Download(user_agent='test') as downloader: tabs = configuration['tabs'] noout = nooutput(tabs) jsonout = jsonoutput(configuration, tabs) outputs = {'gsheets': noout, 'excel': noout, 'json': jsonout} get_indicators(configuration, downloader, outputs, tabs, scrapers=['ifi', 'who', 'covid_trend']) filepath = jsonout.save(tempdir) assert_files_same(filepath, join(folder, 'test_tabular.json'))
def test_save_json(self, tmpdir, saverfolder, filename, pretty, sortkeys): test_path = join(str(tmpdir), filename) ref_path = join(saverfolder, filename) save_json( TestLoader.json_to_write, test_path, pretty=pretty, sortkeys=sortkeys, ) assert_files_same(ref_path, test_path) dct = json.loads(json.dumps(TestLoader.json_to_write)) save_json(dct, test_path, pretty=pretty, sortkeys=sortkeys) assert_files_same(ref_path, test_path)
def test_generate_dataset_and_showcase(self, configuration, data): with temp_dir("ucdp") as folder: countries, headers, countriesdata = data dataset, showcase = generate_dataset_and_showcase( folder, countries[1], countriesdata["BGD"], headers) assert dataset == TestUCDP.dataset resources = dataset.get_resources() assert resources == TestUCDP.resources file = "conflict_data_BGD.csv" assert_files_same(join("tests", "fixtures", file), join(folder, file)) file = "qc_conflict_data_BGD.csv" assert_files_same(join("tests", "fixtures", file), join(folder, file)) assert showcase == { "name": "ucdp-data-for-bangladesh-showcase", "title": "Bangladesh - Data on Conflict Events", "notes": "Conflict Data Dashboard for Bangladesh", "url": "https://ucdp.uu.se/#country/771", "image_url": "https://pbs.twimg.com/profile_images/832251660718178304/y-LWa5iK_200x200.jpg", "tags": [ { "name": "hxl", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "violence and conflict", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "protests", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "security incidents", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, ], }
def test_generate_dataset_and_showcase(self, configuration, data): with temp_dir('ucdp') as folder: countries, headers, countriesdata = data dataset, showcase = generate_dataset_and_showcase( folder, countries[1], countriesdata['BGD'], headers) assert dataset == TestUCDP.dataset resources = dataset.get_resources() assert resources == TestUCDP.resources file = 'conflict_data_BGD.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file)) file = 'qc_conflict_data_BGD.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file)) assert showcase == { 'name': 'ucdp-data-for-bangladesh-showcase', 'title': 'Bangladesh - Conflict Data', 'notes': 'Conflict Data Dashboard for Bangladesh', 'url': 'https://ucdp.uu.se/#country/771', 'image_url': 'https://pbs.twimg.com/profile_images/832251660718178304/y-LWa5iK_200x200.jpg', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'protests', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'security incidents', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }] }
def test_ssd_acled(self, folder, today, ssd_country_list, valid_ssd_adm2_names, replace_ssd_values): resource_updates = dict() filename = 'South_Sudan_Recent_Conflict_Events.csv' expected_events = join('tests', 'fixtures', filename) actual_events = join(folder, filename) resource_updates['acled_events'] = {'path': actual_events} filename = 'South_Sudan_Recent_Conflict_Event_Total_Fatalities.csv' expected_fatalities = join('tests', 'fixtures', filename) actual_fatalities = join(folder, filename) resource_updates['acled_fatalities'] = {'path': actual_fatalities} update_ssd_acled( today, 'https://raw.githubusercontent.com/mcarans/hdxscraper-mapexplorer/master/tests/fixtures/ACLEDSouthSudan.csv?', ssd_country_list, valid_ssd_adm2_names, replace_ssd_values, resource_updates) assert_files_same(expected_events, actual_events) assert_files_same(expected_fatalities, actual_fatalities)
def test_get_indicators(self, configuration, folder): configuration = Configuration.read() countries = configuration['countries'] palestine_country_code, _ = Country.get_iso3_country_code_fuzzy( 'Palestine') df_indicators, df_timeseries, df_cumulative = get_indicators( folder, countries, palestine_country_code, True) with temp_dir('PA-COVID-TEST') as outdir: ifilename = 'indicators.csv' ioutputfile = join(outdir, ifilename) df_indicators.to_csv(ioutputfile) tfilename = 'timeseries.csv' toutputfile = join(outdir, tfilename) df_timeseries.to_csv(toutputfile) cfilename = 'cumulative.csv' coutputfile = join(outdir, cfilename) df_cumulative.to_csv(coutputfile) assert_files_same(join(folder, ifilename), ioutputfile) assert_files_same(join(folder, tfilename), toutputfile) assert_files_same(join(folder, cfilename), coutputfile)
def test_generate_datasets_and_showcase(self, configuration, downloader): with temp_dir('DHS') as folder: dataset, subdataset, showcase, bites_disabled = \ generate_datasets_and_showcase(configuration, 'http://haha/', downloader, folder, TestDHS.country, TestDHS.tags) assert dataset == TestDHS.dataset resources = dataset.get_resources() assert resources == TestDHS.resources assert subdataset == TestDHS.subdataset assert subdataset.get_resources() == TestDHS.subresources assert showcase == {'name': 'dhs-data-for-afghanistan-showcase', 'title': 'Final Report', 'notes': 'Afghanistan Demographic and Health Survey 2015', 'url': 'https://www.dhsprogram.com/pubs/pdf/FR323/FR323.pdf', 'image_url': 'https://www.dhsprogram.com/publications/images/thumbnails/FR323.jpg', 'tags': [{'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}, {'name': 'health', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}, {'name': 'demographics', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87'}]} assert bites_disabled == {'national': [False, False, False], 'subnational': [False, False, False]} file = 'DHS Quickstats_national_AFG.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file)) file = 'DHS Mobile_national_AFG.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file)) file = 'DHS Quickstats_subnational_AFG.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file))
def test_generate_dataset_and_showcase(self, configuration, downloader): with temp_dir("faostat-test") as folder: filelist_url = configuration["filelist_url"] showcase_base_url = configuration["showcase_base_url"] ( dataset, showcase, bites_disabled, qc_indicators, ) = generate_dataset_and_showcase( "Food Security", TestFaostat.indicatorsets, TestFaostat.country, TestFaostat.countrymapping, showcase_base_url, filelist_url, downloader, folder, ) assert dataset == { "name": "faostat-food-security-indicators-for-afghanistan", "title": "Afghanistan - Food Security Indicators", "notes": "Food Security Indicators for Afghanistan.\n\nContains data from the FAOSTAT [bulk data service](http://lala/datasets_E.json).", "maintainer": "196196be-6037-4488-8b71-d786adf4c081", "owner_org": "ed727a5b-3e6e-4cd6-b97e-4a71532085e6", "data_update_frequency": "365", "subnational": "0", "tags": [ { "name": "hxl", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "indicators", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "food security", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, ], "groups": [{ "name": "afg" }], "dataset_date": "[1999-01-01T00:00:00 TO 2014-12-31T00:00:00]", } resources = dataset.get_resources() assert resources == [ { "name": "Suite of Food Security Indicators for Afghanistan", "description": "*Suite of Food Security Indicators:*\nFor detailed description of the indicators below see attached document: Average Dietary Supply Adequacy;...", "format": "csv", "resource_type": "file.upload", "url_type": "upload", }, { "name": "QuickCharts-Suite of Food Security Indicators for Afghanistan", "description": "Cut down data for QuickCharts", "format": "csv", "resource_type": "file.upload", "url_type": "upload", }, ] assert showcase == { "name": "faostat-food-security-indicators-for-afghanistan-showcase", "title": "Afghanistan - Food Security Indicators", "notes": "Food Security Data Dashboard for Afghanistan", "url": "http://www.fao.org/faostat/en/#country/AFG", "image_url": "https://pbs.twimg.com/profile_images/1375385494167691269/Bc49-Yx8_400x400.jpg", "tags": [ { "name": "hxl", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "indicators", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, { "name": "food security", "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", }, ], } assert bites_disabled == [False, True, True] assert qc_indicators == [ { "code": "21010", "title": "Average dietary energy supply adequacy", "unit": "Percentage", }, { "code": "210041", "title": "Prevalence of undernourishment", "unit": "Percentage", }, { "code": "21034", "title": "Percentage of arable land equipped for irrigation", "unit": "Percentage", }, ] file = "Suite of Food Security Indicators_AFG.csv" assert_files_same(join("tests", "fixtures", file), join(folder, file)) file = f"qc_{file}" assert_files_same(join("tests", "fixtures", file), join(folder, file))
def test_generate_datasets_and_showcase(self, configuration, downloader): with temp_dir('idmc') as folder: # indicator dataset test indicators = Configuration.read()['indicators'] tags = Configuration.read()['tags'] datasets, showcase, headersdata, countriesdata = generate_indicator_datasets_and_showcase( downloader, folder, indicators, tags) assert datasets == { 'displacement_data': { 'name': 'idmc-internally-displaced-persons-idps', 'title': 'Internally displaced persons - IDPs', 'maintainer': '196196be-6037-4488-8b71-d786adf4c081', 'owner_org': '647d9d8c-4cac-4c33-b639-649aad1c2893', 'data_update_frequency': '365', 'subnational': '0', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }], 'notes': "Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).", 'methodology_other': 'Methodology', 'caveats': 'Caveats', 'groups': [{ 'name': 'world' }], 'dataset_date': '01/01/2008-12/31/2018' }, 'disaster_data': { 'name': 'idmc-internally-displaced-persons-idps-new-displacement-associated-with-disasters', 'title': 'Internally displaced persons - IDPs (new displacement associated with disasters)', 'maintainer': '196196be-6037-4488-8b71-d786adf4c081', 'owner_org': '647d9d8c-4cac-4c33-b639-649aad1c2893', 'data_update_frequency': '365', 'subnational': '0', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }], 'notes': "Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).", 'methodology_other': 'Methodology', 'caveats': 'Caveats', 'groups': [{ 'name': 'world' }], 'dataset_date': '01/01/2008-12/31/2018' } } resources = datasets['displacement_data'].get_resources() assert resources == [{ 'description': 'Internally displaced persons - IDPs', 'format': 'csv', 'name': 'displacement_data', 'resource_type': 'file.upload', 'url_type': 'upload' }] resource_name = '%s.%s' % (resources[0]['name'], resources[0]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) resources = datasets['disaster_data'].get_resources() resource_name = '%s.%s' % (resources[0]['name'], resources[0]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) assert resources == [{ 'description': 'Internally displaced persons - IDPs (new displacement associated with disasters)', 'format': 'csv', 'name': 'disaster_data', 'resource_type': 'file.upload', 'url_type': 'upload' }] assert showcase == { 'image_url': 'http://www.internal-displacement.org/global-report/grid2018/img/ogimage.jpg', 'name': 'idmc-global-report-on-internal-displacement', 'notes': 'Click the image on the right to go to the IDMC Global Report on Internal Displacement', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }], 'title': 'IDMC Global Report on Internal Displacement', 'url': 'http://www.internal-displacement.org/global-report/grid2018/' } # country datasets tests dataset, showcase, disables_bites = generate_country_dataset_and_showcase( downloader, folder, headersdata, 'AFG', countriesdata['AFG'], datasets, tags) assert dataset == TestIDMC.afg_dataset resources = dataset.get_resources() assert resources == [{ 'description': 'Internally displaced persons - IDPs for Afghanistan', 'format': 'csv', 'name': 'displacement_data', 'resource_type': 'file.upload', 'url_type': 'upload' }, { 'description': 'Internally displaced persons - IDPs (new displacement associated with disasters) for Afghanistan', 'format': 'csv', 'name': 'disaster_data', 'resource_type': 'file.upload', 'url_type': 'upload' }] resource_name = '%s.%s' % (resources[0]['name'], resources[0]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) resource_name = '%s.%s' % (resources[1]['name'], resources[1]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) assert showcase == { 'name': 'idmc-idp-data-for-afghanistan-showcase', 'title': 'IDMC Afghanistan Summary Page', 'notes': 'Click the image on the right to go to the IDMC summary page for the Afghanistan dataset', 'url': 'http://www.internal-displacement.org/countries/Afghanistan/', 'image_url': 'http://www.internal-displacement.org/sites/default/files/logo_0.png', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }] } assert disables_bites == [False, False, False] dataset, showcase, disables_bites = generate_country_dataset_and_showcase( downloader, folder, headersdata, 'TZA', countriesdata['TZA'], datasets, tags) assert dataset == { 'name': 'idmc-idp-data-for-united-republic-of-tanzania', 'title': 'United Republic of Tanzania - Internally displaced persons - IDPs', 'maintainer': '196196be-6037-4488-8b71-d786adf4c081', 'owner_org': '647d9d8c-4cac-4c33-b639-649aad1c2893', 'data_update_frequency': '365', 'subnational': '0', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }], 'groups': [{ 'name': 'tza' }], 'notes': "Description\n\nContains data from IDMC's [Global Internal Displacement Database](http://www.internal-displacement.org/database/displacement-data).", 'methodology_other': '', 'caveats': '', 'dataset_date': '01/01/2011-12/31/2012' } resources = dataset.get_resources() assert resources == [{ 'description': 'Internally displaced persons - IDPs for United Republic of Tanzania', 'format': 'csv', 'name': 'displacement_data', 'resource_type': 'file.upload', 'url_type': 'upload' }, { 'description': 'Internally displaced persons - IDPs (new displacement associated with disasters) for United Republic of Tanzania', 'format': 'csv', 'name': 'disaster_data', 'resource_type': 'file.upload', 'url_type': 'upload' }] resource_name = '%s.%s' % (resources[0]['name'], resources[0]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) resource_name = '%s.%s' % (resources[1]['name'], resources[1]['format']) expected_file = join('tests', 'fixtures', resource_name) actual_file = join(folder, resource_name) assert_files_same(expected_file, actual_file) assert showcase == { 'name': 'idmc-idp-data-for-united-republic-of-tanzania-showcase', 'title': 'IDMC United Republic of Tanzania Summary Page', 'notes': 'Click the image on the right to go to the IDMC summary page for the United Republic of Tanzania dataset', 'url': 'http://www.internal-displacement.org/countries/Tanzania/', 'image_url': 'http://www.internal-displacement.org/sites/default/files/logo_0.png', 'tags': [{ 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'displacement', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'internally displaced persons - idp', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'violence and conflict', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }] } assert disables_bites == [True, True, False] dataset, showcase, disables_bites = generate_country_dataset_and_showcase( downloader, folder, headersdata, 'AB9', countriesdata['AB9'], datasets, tags) assert dataset is None assert showcase is None assert disables_bites is None
def test_generate_dataset_and_showcase(self, configuration): with temp_dir('HDRO') as folder: qc_indicators = configuration['qc_indicators'] dataset, showcase, bites_disabled = generate_dataset_and_showcase( folder, 'AFG', hdro_data, qc_indicators) assert dataset == { 'name': 'hdro-data-for-afghanistan', 'title': 'Afghanistan - Human Development Indicators', 'maintainer': '872427e4-7e9b-44d6-8c58-30d5052a00a2', 'owner_org': '89ebe982-abe9-4748-9dde-cf04632757d6', 'data_update_frequency': '365', 'subnational': '0', 'groups': [{ 'name': 'afg' }], 'tags': [{ 'name': 'health', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'education', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'socioeconomic', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'demographics', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'development', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'indicators', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }], 'dataset_date': '[2008-01-01T00:00:00 TO 2019-12-31T00:00:00]' } resources = dataset.get_resources() assert resources == [{ 'name': 'Human Development Indicators for Afghanistan', 'description': 'Human development data with HXL tags', 'format': 'csv', 'resource_type': 'file.upload', 'url_type': 'upload' }, { 'name': 'QuickCharts-Human Development Indicators for Afghanistan', 'description': 'Cut down data for QuickCharts', 'format': 'csv', 'resource_type': 'file.upload', 'url_type': 'upload' }] assert showcase == { 'name': 'hdro-data-for-afghanistan-showcase', 'title': 'Indicators for Afghanistan', 'notes': 'Human Development indicators for Afghanistan', 'url': 'http://hdr.undp.org/en/countries/profiles/AFG', 'image_url': 'https://s1.stabroeknews.com/images/2019/12/undp.jpg', 'tags': [{ 'name': 'health', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'education', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'socioeconomic', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'demographics', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'development', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'indicators', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }, { 'name': 'hxl', 'vocabulary_id': '4e61d464-4943-4e97-973a-84673c1aaa87' }] } assert bites_disabled == [False, False, True] file = 'hdro_indicators_AFG.csv' assert_files_same(join('tests', 'fixtures', file), join(folder, file)) file = 'qc_%s' % file assert_files_same(join('tests', 'fixtures', file), join(folder, file))
def test_run(self, configuration, fixtures_dir, input_dir): with temp_dir('TestIATIViz', delete_on_success=True, delete_on_failure=False) as tempdir: with Download(user_agent='test') as downloader: retriever = Retrieve(downloader, tempdir, input_dir, tempdir, save=False, use_saved=True) wfp = WFPFood(configuration, None, retriever) countries = wfp.get_countries() assert len(countries) == 291 assert countries[100:102] == [{ 'iso3': 'GTM', 'name': 'Guatemala' }, { 'iso3': 'GUF', 'name': 'French Guiana' }] wfp.build_mappings() dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase( 'COG', tempdir) assert dataset == { 'name': 'wfp-food-prices-for-congo', 'title': 'Congo - Food Prices', 'maintainer': 'f1921552-8c3e-47e9-9804-579b14a83ee3', 'owner_org': '3ecac442-7fed-448d-8f78-b385ef6f84e7', 'data_update_frequency': '7', 'groups': [{ 'name': 'cog' }], 'subnational': '1', 'tags': [{ 'name': 'commodities', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'prices', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'markets', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'hxl', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }], 'dataset_source': 'CARITAS, GOV, Gvt, National Institute Of Statistics (INS), WFP', 'dataset_date': '[2011-01-15T00:00:00 TO 2020-07-15T00:00:00]' } assert showcase == { 'name': 'wfp-food-prices-for-congo-showcase', 'title': 'Congo - Food Prices showcase', 'notes': 'Congo food prices data from World Food Programme displayed through VAM Economic Explorer', 'url': 'http://dataviz.vam.wfp.org/economic_explorer/prices?iso3=COG', 'image_url': 'http://dataviz.vam.wfp.org/_images/home/3_economic.jpg', 'tags': [{ 'name': 'commodities', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'prices', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'markets', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }, { 'name': 'hxl', 'vocabulary_id': 'b891512e-9516-4bf5-962a-7a289772a2a1' }] } assert qc_indicators == [{ 'code': 'Brazzaville-Brazzaville-Total-Groundnuts (shelled)-KG-XAF', 'title': 'Price of Groundnuts (shelled) in Total', 'unit': 'Currency XAF', 'description': 'Price of Groundnuts (shelled) (XAF/KG) in Brazzaville/Total', 'code_col': '#meta+code', 'value_col': '#value', 'date_col': '#date' }, { 'code': 'Point-Noire-Pointe-Noire-Grand marché/Fond Ntié-Ntié/Nkouikou-Oil (vegetable)-L-XAF', 'title': 'Price of Oil (vegetable) in Grand marché/Fond Ntié-Ntié/Nkouikou', 'unit': 'Currency XAF', 'description': 'Price of Oil (vegetable) (XAF/L) in Point-Noire/Pointe-Noire/Grand marché/Fond Ntié-Ntié/Nkouikou', 'code_col': '#meta+code', 'value_col': '#value', 'date_col': '#date' }, { 'code': 'Pool-Kinkala-Kinkala-Rice (mixed, low quality)-KG-XAF', 'title': 'Price of Rice (mixed, low quality) in Kinkala', 'unit': 'Currency XAF', 'description': 'Price of Rice (mixed, low quality) (XAF/KG) in Pool/Kinkala', 'code_col': '#meta+code', 'value_col': '#value', 'date_col': '#date' }] for filename in ('wfp_food_prices_cog', 'wfp_food_prices_cog_qc'): csv_filename = f'{filename}.csv' expected_file = join(fixtures_dir, csv_filename) actual_file = join(tempdir, csv_filename) assert_files_same(expected_file, actual_file)