def generate_dataset(configuration): url = configuration['base_url'] + configuration['api'] loaData.writeData(url) name = 'Africa health facilities' title = 'Africa health facilities data' slugified_name = slugify(name).lower() dataset = Dataset(configuration, {}) dataset['name'] = slugified_name dataset['title'] = title date = time.strftime("%d/%m/%Y") dataset['dataset_date'] = date dataset.add_continent_location('AF') rName = "sen-healthfacilities" resource = Resource() resource['name'] = rName resource['format'] = 'geojson' resource['url'] = url resource['description'] = configuration['base_url'] resource['url_type'] = 'api' resource['resource_type'] = 'api' resource.set_file_to_upload(configuration['data_folder'] + 'sen-healthfacilities.geojson') dataset.add_update_resource(resource) return dataset
def test_update_in_hdx(self, configuration, post_update): dataset = Dataset() dataset['id'] = 'NOTEXIST' with pytest.raises(HDXError): dataset.update_in_hdx() dataset['name'] = 'LALA' with pytest.raises(HDXError): dataset.update_in_hdx() dataset = Dataset.read_from_hdx('TEST1') assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d' assert dataset['dataset_date'] == '06/04/2016' dataset['dataset_date'] = '02/26/2016' dataset['id'] = 'TEST1' dataset['name'] = 'MyDataset1' dataset.update_in_hdx() assert dataset['id'] == 'TEST1' assert dataset['dataset_date'] == '02/26/2016' dataset['id'] = 'NOTEXIST' with pytest.raises(HDXError): dataset.update_in_hdx() del dataset['id'] with pytest.raises(HDXError): dataset.update_in_hdx() dataset_data = copy.deepcopy(TestDataset.dataset_data) gallery_data = copy.deepcopy(TestDataset.gallery_data) dataset_data['name'] = 'MyDataset1' dataset_data['id'] = 'TEST1' dataset = Dataset(dataset_data) dataset.add_update_gallery(gallery_data) dataset.create_in_hdx() assert dataset['id'] == 'TEST1' assert dataset['dataset_date'] == '03/23/2016' assert len(dataset.resources) == 2 assert len(dataset.gallery) == 1 dataset.update_in_hdx() assert len(dataset.resources) == 2 assert len(dataset.gallery) == 1 dataset = Dataset.read_from_hdx('TEST4') del gallery_data[0]['id'] dataset.add_update_gallery(gallery_data) dataset['id'] = 'TEST4' dataset.update_in_hdx() assert len(dataset.resources) == 2 assert len(dataset.gallery) == 1 dataset = Dataset.read_from_hdx('TEST4') resources_data = copy.deepcopy(TestDataset.resources_data) resource = Resource(resources_data[0]) file = tempfile.NamedTemporaryFile(delete=False) resource.set_file_to_upload(file.name) dataset.add_update_resource(resource) dataset.update_in_hdx() os.unlink(file.name) assert len(dataset.resources) == 2 assert len(dataset.gallery) == 0
def test_check_url_filetoupload(self, configuration): resource_data = copy.deepcopy(TestResource.resource_data) resource = Resource(resource_data) resource.check_url_filetoupload() resource.set_file_to_upload('abc') resource.check_url_filetoupload() resource['url'] = 'lala' with pytest.raises(HDXError): resource.check_url_filetoupload()
def test_check_url_filetoupload(self, configuration): resource_data = copy.deepcopy(TestResource.resource_data) resource = Resource(resource_data) resource.check_url_filetoupload() resource.set_file_to_upload('abc') resource.check_url_filetoupload() resource['url'] = 'lala' with pytest.raises(HDXError): resource.check_url_filetoupload()
def test_update_in_hdx(self, configuration, post_update): resource = Resource() resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() resource['name'] = 'LALA' with pytest.raises(HDXError): resource.update_in_hdx() resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811') assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5' assert resource.get_file_type() == 'csv' resource.set_file_type('XLSX') resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811' resource['name'] = 'MyResource1' resource.update_in_hdx() assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811' assert resource['format'] == 'xlsx' assert resource.get_file_type() == 'xlsx' assert resource['url_type'] == 'api' assert resource['resource_type'] == 'api' assert resource[ 'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv' assert resource['state'] == 'active' filetoupload = join('tests', 'fixtures', 'test_data.csv') resource.set_file_to_upload(filetoupload) resource.update_in_hdx() assert resource['url_type'] == 'upload' assert resource['resource_type'] == 'file.upload' assert resource[ 'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv' assert resource['state'] == 'active' resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() del resource['id'] with pytest.raises(HDXError): resource.update_in_hdx() resource.data = dict() with pytest.raises(HDXError): resource.update_in_hdx() resource_data = copy.deepcopy(TestResource.resource_data) resource_data['name'] = 'MyResource1' resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811' resource = Resource(resource_data) resource.create_in_hdx() assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811' assert resource.get_file_type() == 'xlsx' assert resource['state'] == 'active'
def test_update_in_hdx(self, configuration, post_update): resource = Resource() resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() resource['name'] = 'LALA' with pytest.raises(HDXError): resource.update_in_hdx() resource = Resource.read_from_hdx('74b74ae1-df0c-4716-829f-4f939a046811') assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5' assert resource.get_file_type() == 'csv' resource.set_file_type('XLSX') resource['id'] = '74b74ae1-df0c-4716-829f-4f939a046811' resource['name'] = 'MyResource1' resource.update_in_hdx() assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811' assert resource['format'] == 'xlsx' assert resource.get_file_type() == 'xlsx' assert resource['url_type'] == 'api' assert resource['resource_type'] == 'api' assert resource[ 'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv' filetoupload = join('tests', 'fixtures', 'test_data.csv') resource.set_file_to_upload(filetoupload) resource.update_in_hdx() assert resource['url_type'] == 'upload' assert resource['resource_type'] == 'file.upload' assert resource[ 'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv' resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() del resource['id'] with pytest.raises(HDXError): resource.update_in_hdx() resource.data = dict() with pytest.raises(HDXError): resource.update_in_hdx() resource_data = copy.deepcopy(TestResource.resource_data) resource_data['name'] = 'MyResource1' resource_data['id'] = '74b74ae1-df0c-4716-829f-4f939a046811' resource = Resource(resource_data) resource.create_in_hdx() assert resource['id'] == '74b74ae1-df0c-4716-829f-4f939a046811' assert resource.get_file_type() == 'xlsx'
def test_add_update_delete_resources(self, configuration, post_delete): dataset_data = copy.deepcopy(TestDataset.dataset_data) resources_data = copy.deepcopy(TestDataset.resources_data) dataset = Dataset(dataset_data) dataset.add_update_resources(resources_data) assert len(dataset.resources) == 2 dataset.delete_resource('NOTEXIST') assert len(dataset.resources) == 2 dataset.delete_resource('de6549d8-268b-4dfe-adaf-a4ae5c8510d5') assert len(dataset.resources) == 1 resources_data = copy.deepcopy(TestDataset.resources_data) resource = Resource(resources_data[0]) resource.set_file_to_upload('lala') dataset.add_update_resource(resource) assert dataset.resources[1].get_file_to_upload() == 'lala'
def generate_dataset_and_showcase(countryName, countryISO2): title = '%s - Demographic, Health, Education and Transport indicators' % countryName logger.info('Creating dataset: %s' % title) name = 'unhabitat-%s-indicators' % countryISO2 slugified_name = slugify(name).lower() dataset = Dataset({ 'name': slugified_name, 'title': title, }) # dataset.set_dataset_date(date, dataset_end_date=) dataset.set_dataset_year_range(1950, 2050) dataset.set_expected_update_frequency('Every year') dataset.set_subnational(1) dataset.add_country_location(getCountryISO3Code(countryISO2)) dataset.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT', 'HXL']) if os.path.isfile('data/indicator_data_' + countryISO2 + '.csv'): resource = Resource() resource['name'] = 'Indicators_data_%s' % countryISO2 resource[ 'description'] = '%s - Demographic, Health, Education and Transport indicators' % countryName resource['format'] = 'csv' resource.set_file_to_upload('data/indicator_data_' + countryISO2 + '.csv') resource.check_required_fields(['group', 'package_id']) dataset.add_update_resource(resource) showcase_name = slugify('unhabitat-%s' % countryName + ' indacators-data').lower() showcase = Showcase({ 'name': showcase_name, 'title': 'Explore %s' % countryName + ' indicators', 'notes': 'Explore %s' % countryName + ' indicators', 'url': 'http://urbandata.unhabitat.org/data-country/?countries=%s' % countryISO2 + '&indicators=total_length_road,rural_population,urban_population_countries,urban_slum_population_countries,population,income_gini_coefficient_countries', 'image_url': 'https://centre.humdata.org/wp-content/uploads/2018/09/unhabitat-showcase.png' }) showcase.add_tags(['EDUCATION', 'POPULATION', 'HEALTH', 'TRANSPORT']) return dataset, showcase
def test_update_in_hdx(self, configuration, post_update): resource = Resource() resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() resource['name'] = 'LALA' with pytest.raises(HDXError): resource.update_in_hdx() resource = Resource.read_from_hdx('TEST1') assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5' assert resource['format'] == 'XLSX' resource['format'] = 'CSV' resource['id'] = 'TEST1' resource['name'] = 'MyResource1' resource.update_in_hdx() assert resource['id'] == 'TEST1' assert resource['format'] == 'CSV' assert resource['url_type'] == 'api' assert resource['resource_type'] == 'api' assert resource[ 'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv' resource.set_file_to_upload('fixtures/test_data.csv') resource.update_in_hdx() assert resource['url_type'] == 'upload' assert resource['resource_type'] == 'file.upload' assert resource[ 'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv' resource['id'] = 'NOTEXIST' with pytest.raises(HDXError): resource.update_in_hdx() del resource['id'] with pytest.raises(HDXError): resource.update_in_hdx() resource_data = copy.deepcopy(TestResource.resource_data) resource_data['name'] = 'MyResource1' resource_data['id'] = 'TEST1' resource = Resource(resource_data) resource.create_in_hdx() assert resource['id'] == 'TEST1' assert resource['format'] == 'xlsx'
def test_create_in_hdx(self, configuration, post_create): resource = Resource() with pytest.raises(HDXError): resource.create_in_hdx() resource['id'] = 'TEST1' resource['name'] = 'LALA' with pytest.raises(HDXError): resource.create_in_hdx() resource_data = copy.deepcopy(TestResource.resource_data) resource = Resource(resource_data) resource.create_in_hdx() assert resource['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5' assert resource['url_type'] == 'api' assert resource['resource_type'] == 'api' assert resource[ 'url'] == 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/test_data.csv' resource_data = copy.deepcopy(TestResource.resource_data) resource = Resource(resource_data) filetoupload = join('tests', 'fixtures', 'test_data.csv') resource.set_file_to_upload(filetoupload) assert resource.get_file_to_upload() == filetoupload resource.create_in_hdx() assert resource['url_type'] == 'upload' assert resource['resource_type'] == 'file.upload' assert resource[ 'url'] == 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv' resource_data['name'] = 'MyResource2' resource = Resource(resource_data) with pytest.raises(HDXError): resource.create_in_hdx() resource_data['name'] = 'MyResource3' resource = Resource(resource_data) with pytest.raises(HDXError): resource.create_in_hdx()
def generate_dataset(configuration, countryName): #showedName = countryName if (countryName == "Ivory Coast"): showedName = "Cote d'Ivoire" name = countryName + '-healthsites' title = countryName + '-healthsites' slugified_name = slugify(name).lower() # dataset = Dataset(configuration, { # }) dataset = Dataset({ 'name': slugified_name, 'title': title, }) # dataset['name'] = slugified_name # dataset['title'] = title #generating the datasets getCountryHealthSites(configuration, countryName) # geojson resource if (os.path.isfile(configuration.read()['data_folder'] + countryName + '.geojson')): rName = countryName + '-healthsites-geojson' geojsonResource = Resource() geojsonResource['name'] = rName geojsonResource['format'] = 'geojson' geojsonResource['url'] = configuration.read()['base_url'] geojsonResource['description'] = countryName + ' healthsites geojson' geojsonResource.set_file_to_upload( configuration.read()['data_folder'] + countryName + '.geojson') geojsonResource.check_required_fields(['group', 'package_id']) dataset.add_update_resource(geojsonResource) #csv resource if (os.path.isfile(configuration.read()['data_folder'] + countryName + '.csv')): resource_csv = Resource() resource_csv['name'] = countryName + '-healthsites-csv' resource_csv['description'] = countryName + ' healthsites csv' resource_csv['format'] = 'csv' resource_csv.set_file_to_upload(configuration.read()['data_folder'] + countryName + '.csv') resource_csv.check_required_fields(['group', 'package_id']) dataset.add_update_resource(resource_csv) # shp resource if (os.path.isfile(configuration.read()['data_folder'] + countryName + "-shapefiles.zip")): resource_shp = Resource() resource_shp['name'] = countryName + '-healthsites-shp' resource_shp['format'] = 'zipped shapefile' resource_shp['description'] = countryName + ' healthsites shapefiles' resource_shp.set_file_to_upload(configuration.read()['data_folder'] + countryName + "-shapefiles.zip") resource_shp.check_required_fields(['group', 'package_id']) dataset.add_update_resource(resource_shp) return dataset
def generateDatasetBykey(key, countryName): metadata = yaml.load(open('config/metadata.yml', 'r')) title = '%s - ' % countryName + metadata[key]['title'] name = metadata[key]['name'] desc = metadata[key]['notes'] slugified_name = slugify(name).lower() dataset = Dataset({ 'name': slugified_name, 'title': title, 'description': desc }) dataset.set_dataset_year_range(1985, 2017) dataset.set_expected_update_frequency('Every year') dataset.set_subnational(1) dataset.add_country_location(countryName) resource = Resource() rName = '' upCountry = countryName.upper() if key == 'education': dataset.add_tag('EDUCATION') rName = 'UNECA %s - Education' % countryName resource.set_file_to_upload('data/%s-education.csv' % upCountry) if key == 'health': dataset.add_tag('health') rName = 'UNECA %s - Health' % countryName resource.set_file_to_upload('data/%s-health.csv' % upCountry) if key == 'population_and_migration': dataset.add_tags(['population', 'migration']) rName = 'UNECA %s - Population and Migration' % countryName resource.set_file_to_upload('data/%s-population_and_migration.csv' % upCountry) resource['name'] = rName resource['description'] = 'UNECA %s data' % countryName resource['format'] = 'csv' # resource.check_required_fields(['notes']) dataset.add_update_resource(resource) print("==================== %s dataset generated ====================" % key) return dataset
def generate_dataset_and_showcase(downloader, countrydata, endpoints_metadata, folder, merge_resources=True, single_dataset=False, split_to_resources_by_column="STAT_UNIT", remove_useless_columns=True): """ https://api.uis.unesco.org/sdmx/data/UNESCO,DEM_ECO/....AU.?format=csv-:-tab-true-y&locale=en&subscription-key=... :param downloader: Downloader object :param countrydata: Country datastructure from UNESCO API :param endpoints_metadata: Endpoint datastructure from UNESCO API :param folder: temporary folder :param merge_resources: if true, merge resources for all time periods :param single_dataset: if true, put all endpoints into a single dataset :param split_to_resources_by_column: split data into multiple resorces (csv) based on a value in the specified column :param remove_useless_columns: :return: generator yielding (dataset, showcase) tuples. It may yield None, None. """ countryiso2 = countrydata['id'] countryname = countrydata['names'][0]['value'] logger.info("Processing %s" % countryname) if countryname[:4] in ['WB: ', 'SDG:', 'MDG:', 'UIS:', 'EFA:'] or countryname[:5] in ['GEMR:', 'AIMS:'] or \ countryname[:7] in ['UNICEF:', 'UNESCO:']: logger.info('Ignoring %s!' % countryname) yield None, None return countryiso3 = Country.get_iso3_from_iso2(countryiso2) if countryiso3 is None: countryiso3, _ = Country.get_iso3_country_code_fuzzy(countryname) if countryiso3 is None: logger.exception('Cannot get iso3 code for %s!' % countryname) yield None, None return logger.info('Matched %s to %s!' % (countryname, countryiso3)) earliest_year = 10000 latest_year = 0 if single_dataset: name = 'UNESCO indicators - %s' % countryname dataset, showcase = create_dataset_showcase( name, countryname, countryiso2, countryiso3, single_dataset=single_dataset) if dataset is None: return for endpoint in sorted(endpoints_metadata): time.sleep(0.2) indicator, structure_url, more_info_url, dimensions = endpoints_metadata[ endpoint] structure_url = structure_url % countryiso2 response = load_safely(downloader, '%s%s' % (structure_url, dataurl_suffix)) json = response.json() if not single_dataset: name = 'UNESCO %s - %s' % (json["structure"]["name"], countryname) dataset, showcase = create_dataset_showcase( name, countryname, countryiso2, countryiso3, single_dataset=single_dataset) if dataset is None: continue observations = json['structure']['dimensions']['observation'] time_periods = dict() for observation in observations: if observation['id'] == 'TIME_PERIOD': for value in observation['values']: time_periods[int(value['id'])] = value['actualObs'] if len(time_periods) == 0: logger.warning('No time periods for endpoint %s for country %s!' % (indicator, countryname)) continue earliest_year = min(earliest_year, *time_periods.keys()) latest_year = max(latest_year, *time_periods.keys()) csv_url = '%sformat=csv' % structure_url description = more_info_url if description != ' ': description = '[Info on %s](%s)' % (indicator, description) description = 'To save, right click download button & click Save Link/Target As \n%s' % description df = None for start_year, end_year in chunk_years(time_periods): if merge_resources: df1 = download_df(downloader, csv_url, start_year, end_year) if df1 is not None: df = df1 if df is None else df.append(df1) else: url_years = '&startPeriod=%d&endPeriod=%d' % (start_year, end_year) resource = { 'name': '%s (%d-%d)' % (indicator, start_year, end_year), 'description': description, 'format': 'csv', 'url': downloader.get_full_url('%s%s' % (csv_url, url_years)) } dataset.add_update_resource(resource) if df is not None: stat = { x["id"]: x["name"] for d in dimensions if d["id"] == "STAT_UNIT" for x in d["values"] } for value, df_part in split_df_by_column( process_df(df), split_to_resources_by_column): file_csv = join( folder, ("UNESCO_%s_%s.csv" % (countryiso3, endpoint + ("" if value is None else "_" + value))).replace( " ", "-").replace(":", "-").replace("/", "-").replace( ",", "-").replace("(", "-").replace(")", "-")) if remove_useless_columns: df_part = remove_useless_columns_from_df(df_part) df_part["country-iso3"] = countryiso3 df_part.iloc[ 0, df_part.columns.get_loc("country-iso3")] = "#country+iso3" df_part["Indicator name"] = value df_part.iloc[0, df_part.columns.get_loc("Indicator name" )] = "#indicator+name" df_part = postprocess_df(df_part) df_part.to_csv(file_csv, index=False) description_part = stat.get( value, 'Info on %s%s' % ("" if value is None else value + " in ", indicator)) resource = Resource({ 'name': value, 'description': description_part }) resource.set_file_type('csv') resource.set_file_to_upload(file_csv) dataset.add_update_resource(resource) if not single_dataset: if dataset is None or len(dataset.get_resources()) == 0: logger.error('No resources created for country %s, %s!' % (countryname, endpoint)) else: dataset.set_dataset_year_range(min(time_periods.keys()), max(time_periods.keys())) yield dataset, showcase if single_dataset: if dataset is None or len(dataset.get_resources()) == 0: logger.error('No resources created for country %s!' % (countryname)) else: dataset.set_dataset_year_range(earliest_year, latest_year) yield dataset, showcase
def generate_joint_dataset_and_showcase(wfpfood_url, downloader, folder, countriesdata): """Generate single joint datasets and showcases containing data for all countries. """ title = 'Global Food Prices Database (WFP)' logger.info('Creating joint dataset: %s' % title) slugified_name = 'wfp-food-prices' df = joint_dataframe(wfpfood_url, downloader, countriesdata) if len(df) <= 1: logger.warning('Dataset "%s" is empty' % title) return None, None dataset = Dataset({'name': slugified_name, 'title': title}) dataset.set_maintainer( "9957c0e9-cd38-40f1-900b-22c91276154b") # Orest Dubay # dataset.set_maintainer("154de241-38d6-47d3-a77f-0a9848a61df3") dataset.set_organization("3ecac442-7fed-448d-8f78-b385ef6f84e7") maxmonth = (100 * df.mp_year + df.mp_month).max() % 100 dataset.set_dataset_date("%04d-01-01" % df.mp_year.min(), "%04d-%02d-15" % (df.mp_year.max(), maxmonth), "%Y-%m-%d") dataset.set_expected_update_frequency("weekly") dataset.add_country_locations(sorted(df.adm0_name.unique())) dataset.add_tags(tags) file_csv = join(folder, "WFPVAM_FoodPrices.csv") df.to_csv(file_csv, index=False) resource = Resource({ 'name': title, 'description': "Word Food Programme – Food Prices Data Source: WFP Vulnerability Analysis and Mapping (VAM)." }) resource.set_file_type('csv') # set the file type to eg. csv resource.set_file_to_upload(file_csv) dataset.add_update_resource(resource) showcase = Showcase({ 'name': '%s-showcase' % slugified_name, 'title': 'Global Food Prices', 'notes': "Interactive data visualisation of WFP's Food Market Prices dataset", 'url': "https://data.humdata.org/organization/wfp#interactive-data", 'image_url': "https://docs.humdata.org/wp-content/uploads/wfp_food_prices_data_viz.gif" }) showcase.add_tags(tags) dataset.update_from_yaml() dataset['notes'] = dataset[ 'notes'] % 'Global Food Prices data from the World Food Programme covering' dataset.create_in_hdx() showcase.create_in_hdx() showcase.add_dataset(dataset) dataset.get_resource().create_datastore_from_yaml_schema( yaml_path="wfp_food_prices.yml", path=file_csv) logger.info('Finished joint dataset') return dataset, showcase
def generate_dataset_and_showcase(wfpfood_url, downloader, folder, countrydata, shortcuts): """Generate datasets and showcases for each country. """ title = '%s - Food Prices' % countrydata['name'] logger.info('Creating dataset: %s' % title) name = 'WFP food prices for %s' % countrydata[ 'name'] # Example name which should be unique so can include organisation name and country slugified_name = slugify(name).lower() df = read_dataframe(wfpfood_url, downloader, countrydata) if len(df) <= 1: logger.warning('Dataset "%s" is empty' % title) return None, None dataset = Dataset({ 'name': slugified_name, 'title': title, "dataset_preview": "resource_id" }) dataset.set_maintainer( "9957c0e9-cd38-40f1-900b-22c91276154b") # Orest Dubay # dataset.set_maintainer("154de241-38d6-47d3-a77f-0a9848a61df3") dataset.set_organization("3ecac442-7fed-448d-8f78-b385ef6f84e7") dataset.set_dataset_date(df.loc[1:].date.min(), df.loc[1:].date.max(), "%Y-%m-%d") dataset.set_expected_update_frequency("weekly") dataset.add_country_location(countrydata["name"]) dataset.set_subnational(True) dataset.add_tags(tags) dataset.add_tag('hxl') file_csv = join( folder, "WFP_food_prices_%s.csv" % countrydata["name"].replace(" ", "-")) df.to_csv(file_csv, index=False) resource = Resource({ 'name': title, "dataset_preview_enabled": "False", 'description': "Food prices data with HXL tags" }) resource.set_file_type('csv') # set the file type to eg. csv resource.set_file_to_upload(file_csv) dataset.add_update_resource(resource) df1 = quickchart_dataframe(df, shortcuts) file_csv = join( folder, "WFP_food_median_prices_%s.csv" % countrydata["name"].replace(" ", "-")) df1.to_csv(file_csv, index=False) resource = Resource({ 'name': '%s - Food Median Prices' % countrydata['name'], "dataset_preview_enabled": "True", 'description': """Food median prices data with HXL tags. Median of all prices for a given commodity observed on different markets is shown, together with the market where it was observed. Data are shortened in multiple ways: - Rather that prices on all markets, only median price across all markets is shown, together with the market where it has been observed. - Only food commodities are displayed (non-food commodities like fuel and wages are not shown). - Only data after %s are shown. Missing data are interpolated. - Column with shorter commodity names "cmnshort" are available to be used as chart labels. - Units are adapted and prices are rescaled in order to yield comparable values (so that they can be displayed and compared in a single chart). Scaling factor is present in scaling column. Label with full commodity name and a unit (with scale if applicable) is in column "label". This reduces the amount of data and allows to make cleaner charts. """ % (df1.loc[1:].date.min()) }) resource.set_file_type('csv') # set the file type to eg. csv resource.set_file_to_upload(file_csv) dataset.add_update_resource(resource) showcase = Showcase({ 'name': '%s-showcase' % slugified_name, 'title': title + " showcase", 'notes': countrydata["name"] + " food prices data from World Food Programme displayed through VAM Economic Explorer", 'url': "http://dataviz.vam.wfp.org/economic_explorer/prices?adm0=" + countrydata["code"], 'image_url': "http://dataviz.vam.wfp.org/_images/home/economic_2-4.jpg" }) showcase.add_tags(tags) return dataset, showcase
def test_create_in_hdx(self, configuration, post_create): dataset = Dataset() with pytest.raises(HDXError): dataset.create_in_hdx() dataset['id'] = 'TEST1' dataset['name'] = 'LALA' with pytest.raises(HDXError): dataset.create_in_hdx() dataset_data = copy.deepcopy(TestDataset.dataset_data) dataset = Dataset(dataset_data) dataset.create_in_hdx() assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d' assert len(dataset.resources) == 2 assert len(dataset.gallery) == 0 dataset_data['name'] = 'MyDataset2' dataset = Dataset(dataset_data) with pytest.raises(HDXError): dataset.create_in_hdx() dataset_data['name'] = 'MyDataset3' dataset = Dataset(dataset_data) with pytest.raises(HDXError): dataset.create_in_hdx() dataset_data = copy.deepcopy(TestDataset.dataset_data) gallery_data = copy.deepcopy(TestDataset.gallery_data) dataset_data['gallery'] = gallery_data with pytest.raises(HDXError): dataset = Dataset(dataset_data) del dataset_data['gallery'] dataset = Dataset(dataset_data) del gallery_data[0]['id'] dataset.add_update_gallery(gallery_data) dataset.create_in_hdx() assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d' assert len(dataset.resources) == 2 assert len(dataset.gallery) == 1 dataset_data = copy.deepcopy(TestDataset.dataset_data) resources_data = copy.deepcopy(TestDataset.resources_data) dataset_data['resources'] = resources_data with pytest.raises(HDXError): dataset = Dataset(dataset_data) del dataset_data['resources'] dataset = Dataset(dataset_data) del resources_data[0]['id'] del resources_data[1]['id'] dataset.add_update_resources(resources_data) dataset.create_in_hdx() assert dataset['id'] == '6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d' assert len(dataset.resources) == 2 assert len(dataset.gallery) == 0 dataset_data = copy.deepcopy(TestDataset.dataset_data) dataset = Dataset(dataset_data) resource = Resource(resources_data[0]) file = tempfile.NamedTemporaryFile(delete=False) resource.set_file_to_upload(file.name) dataset.add_update_resource(resource) dataset.create_in_hdx() os.unlink(file.name) assert len(dataset.resources) == 2 assert len(dataset.gallery) == 0