def test_project_configuration_dict(self, hdx_key_file): actual_configuration = Configuration.create(hdx_key_file=hdx_key_file) expected_configuration = { 'api_key': '12345', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'hdx_demo_site': { 'url': 'https://demo-data.humdata.org/', 'username': '******', 'password': '******' }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org/', 'username': '******', 'password': '******' }, 'hdx_feature_site': { 'url': 'https://feature-data.humdata.org/', 'username': '******', 'password': '******' }, 'dataset': {'required_fields': [ 'name', 'private', 'title', 'notes', 'dataset_source', 'owner_org', 'dataset_date', 'groups', 'license_id', 'methodology', 'data_update_frequency' ]}, 'resource': {'dataset_id': 'package_id', 'required_fields': ['package_id', 'name', 'format', 'url', 'description', 'url_type', 'resource_type' ]}, 'galleryitem': {'dataset_id': 'dataset_id', 'required_fields': [ 'dataset_id', 'title', 'type', 'description', 'url', 'image_url', ], 'ignore_dataset_id_on_update': True}, } assert actual_configuration == expected_configuration actual_configuration = Configuration.create(hdx_key_file=hdx_key_file, project_config_dict={'abc': '123'}) expected_configuration['abc'] = '123' assert actual_configuration == expected_configuration
def facade(projectmainfn: Callable[[None], None], **kwargs) -> bool: """Facade that handles ScraperWiki and calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: bool: True = success, False = failure """ try: # # Setting up configuration # configuration = Configuration.create(**kwargs) logger.info('--------------------------------------------------') logger.info('> HDX Site: %s' % configuration.get_hdx_site_url()) projectmainfn() except Exception as e: logger.critical(e, exc_info=True) scraperwiki.status('error', 'Run failed: %s' % sys.exc_info()[0]) return False logger.info('Run completed successfully.\n') scraperwiki.status('ok') return True
def test_hdx_configuration_json(self, hdx_key_file, project_config_yaml): hdx_config_json = join('tests', 'fixtures', 'config', 'hdx_config.json') actual_configuration = Configuration.create(hdx_key_file=hdx_key_file, hdx_config_json=hdx_config_json, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org/', 'username': '******', 'password': '******' }, 'dataset': {'required_fields': [ 'name', 'dataset_date', ]}, 'resource': {'dataset_id': 'package_id', 'required_fields': ['name', 'description' ]}, 'galleryitem': {'dataset_id': 'dataset_id', 'required_fields': [ 'dataset_id', ],}, } assert actual_configuration == expected_configuration
def test_get_hdx_key_site(self, hdx_key_file, project_config_yaml): actual_configuration = Configuration.create(hdx_site='prod', hdx_key_file=hdx_key_file, hdx_config_dict={}, project_config_yaml=project_config_yaml) assert actual_configuration.get_api_key() == '12345' assert actual_configuration.get_hdx_site_url() == 'https://data.humdata.org/' assert actual_configuration._get_credentials() == ('', '')
def get_hdx_contents(country, from_datetime, to_datetime): numArticles = 0 config = Configuration.create(hdx_site='prod', hdx_read_only=True) datasets = Dataset.search_in_hdx(country, sort="metadata_modified desc", rows=1) file = open("hdx_contents.txt", "a") for dataset in datasets: file.write("<span style='font-size:16px'>Relevant New Dataset relating to " + country + " on HDX</span><br><br>") if dataset["url"]: file.write("<a href='" + str(dataset["url"]).encode("utf-8") + "''> <span style='font-size:16px'>" + str(dataset["title"]).encode("utf-8") + "</span> </a> <br>") else: file.write("<span style='font-size:16px'>" + str(dataset["title"]).encode("utf-8") + ": No Link Available.</span> <br>") file.write("<span style='font-size:14px'>" + dataset["notes"].encode("utf-8") + "</span>") numArticles += 1 print "writing hdx" file.close() return numArticles
def facade(projectmainfn: Callable[[None], None], **kwargs) -> None: """Facade to simplify project setup that calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: None """ # # Setting up configuration # configuration = Configuration.create(**kwargs) logger.info('--------------------------------------------------') logger.info('> HDX Site: %s' % configuration.get_hdx_site_url()) projectmainfn()
def test_hdx_configuration_dict(self, hdx_key_file, project_config_yaml): actual_configuration = Configuration.create(hdx_site='prod', hdx_key_file=hdx_key_file, hdx_config_dict={ 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'XYZ': {'567': 987} }, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'XYZ': {'567': 987} } assert actual_configuration == expected_configuration
import os import sys #import pandas as pd from hdx.configuration import Configuration from hdx.data.dataset import Dataset old_stdout = sys.stdout log_file = open("C:/Users/JannisV/Rode Kruis/CP data/FME/Admin_data/message_dataset.log","w") sys.stdout = log_file Configuration.create(hdx_site='prod', hdx_read_only=True) tag='administrative boundaries' datatype='zipped shapefile' downloadpath='C:/Users/JannisV/Rode Kruis/CP data/FME/Admin_data/zipfiles/api/' datasets = Dataset.search_in_hdx('',fq='tags:admin*',rows=10) #print(datasets) for dataset in datasets: if tag in dataset.get_tags(): resources = dataset.get_resources() for resource in resources: if resource['format'] == datatype: print(resources) #folder = downloadpath+dataset['name'] #if not os.path.exists(folder): # os.makedirs(folder) #url, path = resource.download(folder) sys.stdout = old_stdout log_file.close()
def configuration(self): hdx_key_file = join('tests', 'fixtures', '.hdxkey') project_config_yaml = join('tests', 'fixtures', 'config', 'project_configuration.yml') Configuration.create(hdx_key_file=hdx_key_file, project_config_yaml=project_config_yaml)
types: - lines select: - name - highway where: highway IS NOT NULL """ if __name__ == '__main__': import json import pprint from hdx.configuration import Configuration import requests Configuration.create( hdx_site=os.getenv('HDX_SITE', 'demo'), hdx_key=os.getenv('HDX_API_KEY'), ) logging.basicConfig() f_s = FeatureSelection(F_S) extent = open('hdx_exports/adm0/GIN_adm0.geojson').read() h = HDXExportSet(dataset_prefix='demodata_test', name='Geopreview Test', extent=extent, feature_selection=f_s, locations=['GIN']) h.sync_resources([ Artifact(['hotosm_roads_gpkg.zip'], 'geopackage', theme='roads'), Artifact(['hotosm_roads_lines_shp.zip'], 'shp', theme='roads') ], 'http://exports-staging.hotosm.org/downloads/4fa2e396-a6bf-4476-829b-c88b953af42c' )
def test_set_hdx_key_value(self, project_config_yaml): actual_configuration = Configuration.create(hdx_site='prod', hdx_key="TEST_HDX_KEY", hdx_config_dict={}, project_config_yaml=project_config_yaml) assert actual_configuration.get_api_key() == 'TEST_HDX_KEY'
from django.contrib.auth.models import Group, User from django.contrib.gis.db import models from django.contrib.gis.geos import GEOSGeometry from django.contrib.postgres.fields import ArrayField from django.db.models.fields import CharField from django.db.models.signals import post_delete, post_save from django.dispatch.dispatcher import receiver from django.utils import timezone from hdx.configuration import Configuration from hdx_exports.hdx_export_set import HDXExportSet from feature_selection.feature_selection import FeatureSelection from utils import FORMAT_NAMES HDX_URL_PREFIX = Configuration.create( hdx_site=settings.HDX_SITE, hdx_key=settings.HDX_API_KEY, ) LOG = logging.getLogger(__name__) # construct the upload path for export config files.. def get_upload_path(instance, filename): """ Construct the path to where the uploaded config file is to be stored. """ configtype = instance.config_type.lower() # sanitize the filename here.. path = 'export/config/{0}/{1}'.format(configtype, instance.filename) return path