def generate_dataset(configuration):
url = configuration['base_url'] + configuration['api']
loaData.writeData(url)
name = 'Africa health facilities'
title = 'Africa health facilities data'
slugified_name = slugify(name).lower()
dataset = Dataset(configuration, {})
dataset['name'] = slugified_name
dataset['title'] = title
date = time.strftime("%d/%m/%Y")
dataset['dataset_date'] = date
dataset.add_continent_location('AF')
rName = "sen-healthfacilities"
resource = Resource()
resource['name'] = rName
resource['format'] = 'geojson'
resource['url'] = url
resource['description'] = configuration['base_url']
resource['url_type'] = 'api'
resource['resource_type'] = 'api'
resource.set_file_to_upload(configuration['data_folder'] +
'sen-healthfacilities.geojson')
dataset.add_update_resource(resource)
return dataset
def test_update_in_hdx(self, configuration, post_update):
dataset = Dataset()
dataset['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset['name'] = 'LALA'
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset = Dataset.read_from_hdx('TEST1')
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
assert dataset['dataset_date'] == '06/04/2016'
dataset['dataset_date'] = '02/26/2016'
dataset['id'] = 'TEST1'
dataset['name'] = 'MyDataset1'
dataset.update_in_hdx()
assert dataset['id'] == 'TEST1'
assert dataset['dataset_date'] == '02/26/2016'
dataset['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
dataset.update_in_hdx()
del dataset['id']
with pytest.raises(HDXError):
dataset.update_in_hdx()
dataset_data = copy.deepcopy(TestDataset.dataset_data)
gallery_data = copy.deepcopy(TestDataset.gallery_data)
dataset_data['name'] = 'MyDataset1'
dataset_data['id'] = 'TEST1'
dataset = Dataset(dataset_data)
dataset.add_update_gallery(gallery_data)
dataset.create_in_hdx()
assert dataset['id'] == 'TEST1'
assert dataset['dataset_date'] == '03/23/2016'
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset.update_in_hdx()
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset = Dataset.read_from_hdx('TEST4')
del gallery_data[0]['id']
dataset.add_update_gallery(gallery_data)
dataset['id'] = 'TEST4'
dataset.update_in_hdx()
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset = Dataset.read_from_hdx('TEST4')
resources_data = copy.deepcopy(TestDataset.resources_data)
resource = Resource(resources_data[0])
file = tempfile.NamedTemporaryFile(delete=False)
resource.set_file_to_upload(file.name)
dataset.add_update_resource(resource)
dataset.update_in_hdx()
os.unlink(file.name)
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 0
def test_check_url_filetoupload(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.set_file_to_upload('abc')
resource.check_url_filetoupload()
resource['url'] = 'lala'
with pytest.raises(HDXError):
resource.check_url_filetoupload()
def test_check_url_filetoupload(self, configuration):
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.check_url_filetoupload()
resource.set_file_to_upload('abc')
resource.check_url_filetoupload()
resource['url'] = 'lala'
with pytest.raises(HDXError):
resource.check_url_filetoupload()
def test_update_in_hdx(self, configuration, post_update):
resource = Resource()
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource.get_file_type() == 'csv'
resource.set_file_type('XLSX')
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource['format'] == 'xlsx'
assert resource.get_file_type() == 'xlsx'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
assert resource['state'] == 'active'
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
resource.update_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
assert resource['state'] == 'active'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource.data = dict()
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource.get_file_type() == 'xlsx'
assert resource['state'] == 'active'
def test_update_in_hdx(self, configuration, post_update):
resource = Resource()
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource.get_file_type() == 'csv'
resource.set_file_type('XLSX')
resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource['format'] == 'xlsx'
assert resource.get_file_type() == 'xlsx'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
resource.update_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource.data = dict()
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811'
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811'
assert resource.get_file_type() == 'xlsx'
def test_add_update_delete_resources(self, configuration, post_delete):
dataset_data = copy.deepcopy(TestDataset.dataset_data)
resources_data = copy.deepcopy(TestDataset.resources_data)
dataset = Dataset(dataset_data)
dataset.add_update_resources(resources_data)
assert len(dataset.resources) == 2
dataset.delete_resource('NOTEXIST')
assert len(dataset.resources) == 2
dataset.delete_resource('de6549d8-268b-4dfe-adaf-a4ae5c8510d5')
assert len(dataset.resources) == 1
resources_data = copy.deepcopy(TestDataset.resources_data)
resource = Resource(resources_data[0])
resource.set_file_to_upload('lala')
dataset.add_update_resource(resource)
assert dataset.resources[1].get_file_to_upload() == 'lala'
def generate_dataset_and_showcase(countryName, countryISO2):
title = '%s - Demographic, Health, Education and Transport indicators' % countryName
logger.info('Creating dataset: %s' % title)
name = 'unhabitat-%s-indicators' % countryISO2
slugified_name = slugify(name).lower()
dataset = Dataset({
'name': slugified_name,
'title': title,
})
# dataset.set_dataset_date(date, dataset_end_date=)
dataset.set_dataset_year_range(1950, 2050)
dataset.set_expected_update_frequency('Every year')
dataset.set_subnational(1)
dataset.add_country_location(getCountryISO3Code(countryISO2))
dataset.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT', 'HXL'])
if os.path.isfile('data/indicator_data_' + countryISO2 + '.csv'):
resource = Resource()
resource['name'] = 'Indicators_data_%s' % countryISO2
resource[
'description'] = '%s - Demographic, Health, Education and Transport indicators' % countryName
resource['format'] = 'csv'
resource.set_file_to_upload('data/indicator_data_' + countryISO2 +
'.csv')
resource.check_required_fields(['group', 'package_id'])
dataset.add_update_resource(resource)
showcase_name = slugify('unhabitat-%s' % countryName +
' indacators-data').lower()
showcase = Showcase({
'name':
showcase_name,
'title':
'Explore %s' % countryName + ' indicators',
'notes':
'Explore %s' % countryName + ' indicators',
'url':
'http://urbandata.unhabitat.org/data-country/?countries=%s' %
countryISO2 +
'&indicators=total_length_road,rural_population,urban_population_countries,urban_slum_population_countries,population,income_gini_coefficient_countries',
'image_url':
'https://centre.humdata.org/wp-content/uploads/2018/09/unhabitat-showcase.png'
})
showcase.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT'])
return dataset, showcase
def test_update_in_hdx(self, configuration, post_update):
resource = Resource()
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.update_in_hdx()
resource = Resource.read_from_hdx('TEST1')
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['format'] == 'XLSX'
resource['format'] = 'CSV'
resource['id'] = 'TEST1'
resource['name'] = 'MyResource1'
resource.update_in_hdx()
assert resource['id'] == 'TEST1'
assert resource['format'] == 'CSV'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
resource.set_file_to_upload('fixtures/test_data.csv')
resource.update_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
resource['id'] = 'NOTEXIST'
with pytest.raises(HDXError):
resource.update_in_hdx()
del resource['id']
with pytest.raises(HDXError):
resource.update_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource_data['name'] = 'MyResource1'
resource_data['id'] = 'TEST1'
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == 'TEST1'
assert resource['format'] == 'xlsx'
def test_create_in_hdx(self, configuration, post_create):
resource = Resource()
with pytest.raises(HDXError):
resource.create_in_hdx()
resource['id'] = 'TEST1'
resource['name'] = 'LALA'
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
resource.create_in_hdx()
assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5'
assert resource['url_type'] == 'api'
assert resource['resource_type'] == 'api'
assert resource[
'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv'
resource_data = copy.deepcopy(TestResource.resource_data)
resource = Resource(resource_data)
filetoupload = join('tests', 'fixtures', 'test_data.csv')
resource.set_file_to_upload(filetoupload)
assert resource.get_file_to_upload() == filetoupload
resource.create_in_hdx()
assert resource['url_type'] == 'upload'
assert resource['resource_type'] == 'file.upload'
assert resource[
'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv'
resource_data['name'] = 'MyResource2'
resource = Resource(resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
resource_data['name'] = 'MyResource3'
resource = Resource(resource_data)
with pytest.raises(HDXError):
resource.create_in_hdx()
def generate_dataset(configuration, countryName):
#showedName = countryName
if (countryName == "Ivory Coast"):
showedName = "Cote d'Ivoire"
name = countryName + '-healthsites'
title = countryName + '-healthsites'
slugified_name = slugify(name).lower()
# dataset = Dataset(configuration, {
# })
dataset = Dataset({
'name': slugified_name,
'title': title,
})
# dataset['name'] = slugified_name
# dataset['title'] = title
#generating the datasets
getCountryHealthSites(configuration, countryName)
# geojson resource
if (os.path.isfile(configuration.read()['data_folder'] + countryName +
'.geojson')):
rName = countryName + '-healthsites-geojson'
geojsonResource = Resource()
geojsonResource['name'] = rName
geojsonResource['format'] = 'geojson'
geojsonResource['url'] = configuration.read()['base_url']
geojsonResource['description'] = countryName + ' healthsites geojson'
geojsonResource.set_file_to_upload(
configuration.read()['data_folder'] + countryName + '.geojson')
geojsonResource.check_required_fields(['group', 'package_id'])
dataset.add_update_resource(geojsonResource)
#csv resource
if (os.path.isfile(configuration.read()['data_folder'] + countryName +
'.csv')):
resource_csv = Resource()
resource_csv['name'] = countryName + '-healthsites-csv'
resource_csv['description'] = countryName + ' healthsites csv'
resource_csv['format'] = 'csv'
resource_csv.set_file_to_upload(configuration.read()['data_folder'] +
countryName + '.csv')
resource_csv.check_required_fields(['group', 'package_id'])
dataset.add_update_resource(resource_csv)
# shp resource
if (os.path.isfile(configuration.read()['data_folder'] + countryName +
"-shapefiles.zip")):
resource_shp = Resource()
resource_shp['name'] = countryName + '-healthsites-shp'
resource_shp['format'] = 'zipped shapefile'
resource_shp['description'] = countryName + ' healthsites shapefiles'
resource_shp.set_file_to_upload(configuration.read()['data_folder'] +
countryName + "-shapefiles.zip")
resource_shp.check_required_fields(['group', 'package_id'])
dataset.add_update_resource(resource_shp)
return dataset
def generateDatasetBykey(key, countryName):
metadata = yaml.load(open('config/metadata.yml', 'r'))
title = '%s - ' % countryName + metadata[key]['title']
name = metadata[key]['name']
desc = metadata[key]['notes']
slugified_name = slugify(name).lower()
dataset = Dataset({
'name': slugified_name,
'title': title,
'description': desc
})
dataset.set_dataset_year_range(1985, 2017)
dataset.set_expected_update_frequency('Every year')
dataset.set_subnational(1)
dataset.add_country_location(countryName)
resource = Resource()
rName = ''
upCountry = countryName.upper()
if key == 'education':
dataset.add_tag('EDUCATION')
rName = 'UNECA %s - Education' % countryName
resource.set_file_to_upload('data/%s-education.csv' % upCountry)
if key == 'health':
dataset.add_tag('health')
rName = 'UNECA %s - Health' % countryName
resource.set_file_to_upload('data/%s-health.csv' % upCountry)
if key == 'population_and_migration':
dataset.add_tags(['population', 'migration'])
rName = 'UNECA %s - Population and Migration' % countryName
resource.set_file_to_upload('data/%s-population_and_migration.csv' %
upCountry)
resource['name'] = rName
resource['description'] = 'UNECA %s data' % countryName
resource['format'] = 'csv'
# resource.check_required_fields(['notes'])
dataset.add_update_resource(resource)
print("==================== %s dataset generated ====================" %
key)
return dataset
def generate_dataset_and_showcase(downloader,
countrydata,
endpoints_metadata,
folder,
merge_resources=True,
single_dataset=False,
split_to_resources_by_column="STAT_UNIT",
remove_useless_columns=True):
"""
https://api.uis.unesco.org/sdmx/data/UNESCO,DEM_ECO/....AU.?format=csv-:-tab-true-y&locale=en&subscription-key=...
:param downloader: Downloader object
:param countrydata: Country datastructure from UNESCO API
:param endpoints_metadata: Endpoint datastructure from UNESCO API
:param folder: temporary folder
:param merge_resources: if true, merge resources for all time periods
:param single_dataset: if true, put all endpoints into a single dataset
:param split_to_resources_by_column: split data into multiple resorces (csv) based on a value in the specified column
:param remove_useless_columns:
:return: generator yielding (dataset, showcase) tuples. It may yield None, None.
"""
countryiso2 = countrydata['id']
countryname = countrydata['names'][0]['value']
logger.info("Processing %s" % countryname)
if countryname[:4] in ['WB: ', 'SDG:', 'MDG:', 'UIS:', 'EFA:'] or countryname[:5] in ['GEMR:', 'AIMS:'] or \
countryname[:7] in ['UNICEF:', 'UNESCO:']:
logger.info('Ignoring %s!' % countryname)
yield None, None
return
countryiso3 = Country.get_iso3_from_iso2(countryiso2)
if countryiso3 is None:
countryiso3, _ = Country.get_iso3_country_code_fuzzy(countryname)
if countryiso3 is None:
logger.exception('Cannot get iso3 code for %s!' % countryname)
yield None, None
return
logger.info('Matched %s to %s!' % (countryname, countryiso3))
earliest_year = 10000
latest_year = 0
if single_dataset:
name = 'UNESCO indicators - %s' % countryname
dataset, showcase = create_dataset_showcase(
name,
countryname,
countryiso2,
countryiso3,
single_dataset=single_dataset)
if dataset is None:
return
for endpoint in sorted(endpoints_metadata):
time.sleep(0.2)
indicator, structure_url, more_info_url, dimensions = endpoints_metadata[
endpoint]
structure_url = structure_url % countryiso2
response = load_safely(downloader,
'%s%s' % (structure_url, dataurl_suffix))
json = response.json()
if not single_dataset:
name = 'UNESCO %s - %s' % (json["structure"]["name"], countryname)
dataset, showcase = create_dataset_showcase(
name,
countryname,
countryiso2,
countryiso3,
single_dataset=single_dataset)
if dataset is None:
continue
observations = json['structure']['dimensions']['observation']
time_periods = dict()
for observation in observations:
if observation['id'] == 'TIME_PERIOD':
for value in observation['values']:
time_periods[int(value['id'])] = value['actualObs']
if len(time_periods) == 0:
logger.warning('No time periods for endpoint %s for country %s!' %
(indicator, countryname))
continue
earliest_year = min(earliest_year, *time_periods.keys())
latest_year = max(latest_year, *time_periods.keys())
csv_url = '%sformat=csv' % structure_url
description = more_info_url
if description != ' ':
description = '[Info on %s](%s)' % (indicator, description)
description = 'To save, right click download button & click Save Link/Target As \n%s' % description
df = None
for start_year, end_year in chunk_years(time_periods):
if merge_resources:
df1 = download_df(downloader, csv_url, start_year, end_year)
if df1 is not None:
df = df1 if df is None else df.append(df1)
else:
url_years = '&startPeriod=%d&endPeriod=%d' % (start_year,
end_year)
resource = {
'name': '%s (%d-%d)' % (indicator, start_year, end_year),
'description': description,
'format': 'csv',
'url':
downloader.get_full_url('%s%s' % (csv_url, url_years))
}
dataset.add_update_resource(resource)
if df is not None:
stat = {
x["id"]: x["name"]
for d in dimensions if d["id"] == "STAT_UNIT"
for x in d["values"]
}
for value, df_part in split_df_by_column(
process_df(df), split_to_resources_by_column):
file_csv = join(
folder,
("UNESCO_%s_%s.csv" %
(countryiso3, endpoint +
("" if value is None else "_" + value))).replace(
" ",
"-").replace(":", "-").replace("/", "-").replace(
",", "-").replace("(", "-").replace(")", "-"))
if remove_useless_columns:
df_part = remove_useless_columns_from_df(df_part)
df_part["country-iso3"] = countryiso3
df_part.iloc[
0,
df_part.columns.get_loc("country-iso3")] = "#country+iso3"
df_part["Indicator name"] = value
df_part.iloc[0, df_part.columns.get_loc("Indicator name"
)] = "#indicator+name"
df_part = postprocess_df(df_part)
df_part.to_csv(file_csv, index=False)
description_part = stat.get(
value, 'Info on %s%s' %
("" if value is None else value + " in ", indicator))
resource = Resource({
'name': value,
'description': description_part
})
resource.set_file_type('csv')
resource.set_file_to_upload(file_csv)
dataset.add_update_resource(resource)
if not single_dataset:
if dataset is None or len(dataset.get_resources()) == 0:
logger.error('No resources created for country %s, %s!' %
(countryname, endpoint))
else:
dataset.set_dataset_year_range(min(time_periods.keys()),
max(time_periods.keys()))
yield dataset, showcase
if single_dataset:
if dataset is None or len(dataset.get_resources()) == 0:
logger.error('No resources created for country %s!' %
(countryname))
else:
dataset.set_dataset_year_range(earliest_year, latest_year)
yield dataset, showcase
def generate_joint_dataset_and_showcase(wfpfood_url, downloader, folder,
countriesdata):
"""Generate single joint datasets and showcases containing data for all countries.
"""
title = 'Global Food Prices Database (WFP)'
logger.info('Creating joint dataset: %s' % title)
slugified_name = 'wfp-food-prices'
df = joint_dataframe(wfpfood_url, downloader, countriesdata)
if len(df) <= 1:
logger.warning('Dataset "%s" is empty' % title)
return None, None
dataset = Dataset({'name': slugified_name, 'title': title})
dataset.set_maintainer(
"9957c0e9-cd38-40f1-900b-22c91276154b") # Orest Dubay
# dataset.set_maintainer("154de241-38d6-47d3-a77f-0a9848a61df3")
dataset.set_organization("3ecac442-7fed-448d-8f78-b385ef6f84e7")
maxmonth = (100 * df.mp_year + df.mp_month).max() % 100
dataset.set_dataset_date("%04d-01-01" % df.mp_year.min(),
"%04d-%02d-15" % (df.mp_year.max(), maxmonth),
"%Y-%m-%d")
dataset.set_expected_update_frequency("weekly")
dataset.add_country_locations(sorted(df.adm0_name.unique()))
dataset.add_tags(tags)
file_csv = join(folder, "WFPVAM_FoodPrices.csv")
df.to_csv(file_csv, index=False)
resource = Resource({
'name':
title,
'description':
"Word Food Programme – Food Prices Data Source: WFP Vulnerability Analysis and Mapping (VAM)."
})
resource.set_file_type('csv') # set the file type to eg. csv
resource.set_file_to_upload(file_csv)
dataset.add_update_resource(resource)
showcase = Showcase({
'name':
'%s-showcase' % slugified_name,
'title':
'Global Food Prices',
'notes':
"Interactive data visualisation of WFP's Food Market Prices dataset",
'url':
"https://data.humdata.org/organization/wfp#interactive-data",
'image_url':
"https://docs.humdata.org/wp-content/uploads/wfp_food_prices_data_viz.gif"
})
showcase.add_tags(tags)
dataset.update_from_yaml()
dataset['notes'] = dataset[
'notes'] % 'Global Food Prices data from the World Food Programme covering'
dataset.create_in_hdx()
showcase.create_in_hdx()
showcase.add_dataset(dataset)
dataset.get_resource().create_datastore_from_yaml_schema(
yaml_path="wfp_food_prices.yml", path=file_csv)
logger.info('Finished joint dataset')
return dataset, showcase
def generate_dataset_and_showcase(wfpfood_url, downloader, folder, countrydata,
shortcuts):
"""Generate datasets and showcases for each country.
"""
title = '%s - Food Prices' % countrydata['name']
logger.info('Creating dataset: %s' % title)
name = 'WFP food prices for %s' % countrydata[
'name'] # Example name which should be unique so can include organisation name and country
slugified_name = slugify(name).lower()
df = read_dataframe(wfpfood_url, downloader, countrydata)
if len(df) <= 1:
logger.warning('Dataset "%s" is empty' % title)
return None, None
dataset = Dataset({
'name': slugified_name,
'title': title,
"dataset_preview": "resource_id"
})
dataset.set_maintainer(
"9957c0e9-cd38-40f1-900b-22c91276154b") # Orest Dubay
# dataset.set_maintainer("154de241-38d6-47d3-a77f-0a9848a61df3")
dataset.set_organization("3ecac442-7fed-448d-8f78-b385ef6f84e7")
dataset.set_dataset_date(df.loc[1:].date.min(), df.loc[1:].date.max(),
"%Y-%m-%d")
dataset.set_expected_update_frequency("weekly")
dataset.add_country_location(countrydata["name"])
dataset.set_subnational(True)
dataset.add_tags(tags)
dataset.add_tag('hxl')
file_csv = join(
folder,
"WFP_food_prices_%s.csv" % countrydata["name"].replace(" ", "-"))
df.to_csv(file_csv, index=False)
resource = Resource({
'name': title,
"dataset_preview_enabled": "False",
'description': "Food prices data with HXL tags"
})
resource.set_file_type('csv') # set the file type to eg. csv
resource.set_file_to_upload(file_csv)
dataset.add_update_resource(resource)
df1 = quickchart_dataframe(df, shortcuts)
file_csv = join(
folder, "WFP_food_median_prices_%s.csv" %
countrydata["name"].replace(" ", "-"))
df1.to_csv(file_csv, index=False)
resource = Resource({
'name':
'%s - Food Median Prices' % countrydata['name'],
"dataset_preview_enabled":
"True",
'description':
"""Food median prices data with HXL tags.
Median of all prices for a given commodity observed on different markets is shown, together with the market where
it was observed. Data are shortened in multiple ways:
- Rather that prices on all markets, only median price across all markets is shown, together with the market
where it has been observed.
- Only food commodities are displayed (non-food commodities like fuel and wages are not shown).
- Only data after %s are shown. Missing data are interpolated.
- Column with shorter commodity names "cmnshort" are available to be used as chart labels.
- Units are adapted and prices are rescaled in order to yield comparable values (so that they
can be displayed and compared in a single chart). Scaling factor is present in scaling column.
Label with full commodity name and a unit (with scale if applicable) is in column "label".
This reduces the amount of data and allows to make cleaner charts.
""" % (df1.loc[1:].date.min())
})
resource.set_file_type('csv') # set the file type to eg. csv
resource.set_file_to_upload(file_csv)
dataset.add_update_resource(resource)
showcase = Showcase({
'name':
'%s-showcase' % slugified_name,
'title':
title + " showcase",
'notes':
countrydata["name"] +
" food prices data from World Food Programme displayed through VAM Economic Explorer",
'url':
"http://dataviz.vam.wfp.org/economic_explorer/prices?adm0=" +
countrydata["code"],
'image_url':
"http://dataviz.vam.wfp.org/_images/home/economic_2-4.jpg"
})
showcase.add_tags(tags)
return dataset, showcase
def test_create_in_hdx(self, configuration, post_create):
dataset = Dataset()
with pytest.raises(HDXError):
dataset.create_in_hdx()
dataset['id'] = 'TEST1'
dataset['name'] = 'LALA'
with pytest.raises(HDXError):
dataset.create_in_hdx()
dataset_data = copy.deepcopy(TestDataset.dataset_data)
dataset = Dataset(dataset_data)
dataset.create_in_hdx()
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 0
dataset_data['name'] = 'MyDataset2'
dataset = Dataset(dataset_data)
with pytest.raises(HDXError):
dataset.create_in_hdx()
dataset_data['name'] = 'MyDataset3'
dataset = Dataset(dataset_data)
with pytest.raises(HDXError):
dataset.create_in_hdx()
dataset_data = copy.deepcopy(TestDataset.dataset_data)
gallery_data = copy.deepcopy(TestDataset.gallery_data)
dataset_data['gallery'] = gallery_data
with pytest.raises(HDXError):
dataset = Dataset(dataset_data)
del dataset_data['gallery']
dataset = Dataset(dataset_data)
del gallery_data[0]['id']
dataset.add_update_gallery(gallery_data)
dataset.create_in_hdx()
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 1
dataset_data = copy.deepcopy(TestDataset.dataset_data)
resources_data = copy.deepcopy(TestDataset.resources_data)
dataset_data['resources'] = resources_data
with pytest.raises(HDXError):
dataset = Dataset(dataset_data)
del dataset_data['resources']
dataset = Dataset(dataset_data)
del resources_data[0]['id']
del resources_data[1]['id']
dataset.add_update_resources(resources_data)
dataset.create_in_hdx()
assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d'
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 0
dataset_data = copy.deepcopy(TestDataset.dataset_data)
dataset = Dataset(dataset_data)
resource = Resource(resources_data[0])
file = tempfile.NamedTemporaryFile(delete=False)
resource.set_file_to_upload(file.name)
dataset.add_update_resource(resource)
dataset.create_in_hdx()
os.unlink(file.name)
assert len(dataset.resources) == 2
assert len(dataset.gallery) == 0