def test_project_configuration_dict(self, hdx_key_file):
actual_configuration = Configuration.create(hdx_key_file=hdx_key_file)
expected_configuration = {
'api_key': '12345',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'hdx_demo_site': {
'url': 'https://demo-data.humdata.org/',
'username': '******',
'password': '******'
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org/',
'username': '******',
'password': '******'
},
'hdx_feature_site': {
'url': 'https://feature-data.humdata.org/',
'username': '******',
'password': '******'
},
'dataset': {'required_fields': [
'name',
'private',
'title',
'notes',
'dataset_source',
'owner_org',
'dataset_date',
'groups',
'license_id',
'methodology',
'data_update_frequency'
]},
'resource': {'dataset_id': 'package_id',
'required_fields': ['package_id', 'name', 'format', 'url', 'description',
'url_type', 'resource_type'
]},
'galleryitem': {'dataset_id': 'dataset_id', 'required_fields': [
'dataset_id',
'title',
'type',
'description',
'url',
'image_url',
], 'ignore_dataset_id_on_update': True},
}
assert actual_configuration == expected_configuration
actual_configuration = Configuration.create(hdx_key_file=hdx_key_file, project_config_dict={'abc': '123'})
expected_configuration['abc'] = '123'
assert actual_configuration == expected_configuration
def facade(projectmainfn: Callable[[None], None], **kwargs) -> bool:
"""Facade that handles ScraperWiki and calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
bool: True = success, False = failure
"""
try:
#
# Setting up configuration
#
configuration = Configuration.create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> HDX Site: %s' % configuration.get_hdx_site_url())
projectmainfn()
except Exception as e:
logger.critical(e, exc_info=True)
scraperwiki.status('error', 'Run failed: %s' % sys.exc_info()[0])
return False
logger.info('Run completed successfully.\n')
scraperwiki.status('ok')
return True
def test_hdx_configuration_json(self, hdx_key_file, project_config_yaml):
hdx_config_json = join('tests', 'fixtures', 'config', 'hdx_config.json')
actual_configuration = Configuration.create(hdx_key_file=hdx_key_file, hdx_config_json=hdx_config_json,
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org/',
'username': '******',
'password': '******'
},
'dataset': {'required_fields': [
'name',
'dataset_date',
]},
'resource': {'dataset_id': 'package_id',
'required_fields': ['name', 'description'
]},
'galleryitem': {'dataset_id': 'dataset_id', 'required_fields': [
'dataset_id',
],},
}
assert actual_configuration == expected_configuration
def test_get_hdx_key_site(self, hdx_key_file, project_config_yaml):
actual_configuration = Configuration.create(hdx_site='prod', hdx_key_file=hdx_key_file,
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert actual_configuration.get_api_key() == '12345'
assert actual_configuration.get_hdx_site_url() == 'https://data.humdata.org/'
assert actual_configuration._get_credentials() == ('', '')
def get_hdx_contents(country, from_datetime, to_datetime):
numArticles = 0
config = Configuration.create(hdx_site='prod', hdx_read_only=True)
datasets = Dataset.search_in_hdx(country, sort="metadata_modified desc", rows=1)
file = open("hdx_contents.txt", "a")
for dataset in datasets:
file.write("<span style='font-size:16px'>Relevant New Dataset relating to " + country + " on HDX</span><br><br>")
if dataset["url"]:
file.write("<a href='" + str(dataset["url"]).encode("utf-8") + "''> <span style='font-size:16px'>" + str(dataset["title"]).encode("utf-8") + "</span> </a> <br>")
else:
file.write("<span style='font-size:16px'>" + str(dataset["title"]).encode("utf-8") + ": No Link Available.</span> <br>")
file.write("<span style='font-size:14px'>" + dataset["notes"].encode("utf-8") + "</span>")
numArticles += 1
print "writing hdx"
file.close()
return numArticles
def facade(projectmainfn: Callable[[None], None], **kwargs) -> None:
"""Facade to simplify project setup that calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
None
"""
#
# Setting up configuration
#
configuration = Configuration.create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> HDX Site: %s' % configuration.get_hdx_site_url())
projectmainfn()
def test_hdx_configuration_dict(self, hdx_key_file, project_config_yaml):
actual_configuration = Configuration.create(hdx_site='prod', hdx_key_file=hdx_key_file,
hdx_config_dict={
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'XYZ': {'567': 987}
},
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'XYZ': {'567': 987}
}
assert actual_configuration == expected_configuration
import os
import sys
#import pandas as pd
from hdx.configuration import Configuration
from hdx.data.dataset import Dataset
old_stdout = sys.stdout
log_file = open("C:/Users/JannisV/Rode Kruis/CP data/FME/Admin_data/message_dataset.log","w")
sys.stdout = log_file
Configuration.create(hdx_site='prod', hdx_read_only=True)
tag='administrative boundaries'
datatype='zipped shapefile'
downloadpath='C:/Users/JannisV/Rode Kruis/CP data/FME/Admin_data/zipfiles/api/'
datasets = Dataset.search_in_hdx('',fq='tags:admin*',rows=10)
#print(datasets)
for dataset in datasets:
if tag in dataset.get_tags():
resources = dataset.get_resources()
for resource in resources:
if resource['format'] == datatype:
print(resources)
#folder = downloadpath+dataset['name']
#if not os.path.exists(folder):
# os.makedirs(folder)
#url, path = resource.download(folder)
sys.stdout = old_stdout
log_file.close()
def configuration(self):
hdx_key_file = join('tests', 'fixtures', '.hdxkey')
project_config_yaml = join('tests', 'fixtures', 'config', 'project_configuration.yml')
Configuration.create(hdx_key_file=hdx_key_file, project_config_yaml=project_config_yaml)
types:
- lines
select:
- name
- highway
where: highway IS NOT NULL
"""
if __name__ == '__main__':
import json
import pprint
from hdx.configuration import Configuration
import requests
Configuration.create(
hdx_site=os.getenv('HDX_SITE', 'demo'),
hdx_key=os.getenv('HDX_API_KEY'),
)
logging.basicConfig()
f_s = FeatureSelection(F_S)
extent = open('hdx_exports/adm0/GIN_adm0.geojson').read()
h = HDXExportSet(dataset_prefix='demodata_test',
name='Geopreview Test',
extent=extent,
feature_selection=f_s,
locations=['GIN'])
h.sync_resources([
Artifact(['hotosm_roads_gpkg.zip'], 'geopackage', theme='roads'),
Artifact(['hotosm_roads_lines_shp.zip'], 'shp', theme='roads')
], 'http://exports-staging.hotosm.org/downloads/4fa2e396-a6bf-4476-829b-c88b953af42c'
)
def test_set_hdx_key_value(self, project_config_yaml):
actual_configuration = Configuration.create(hdx_site='prod', hdx_key="TEST_HDX_KEY",
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert actual_configuration.get_api_key() == 'TEST_HDX_KEY'
from django.contrib.auth.models import Group, User
from django.contrib.gis.db import models
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.postgres.fields import ArrayField
from django.db.models.fields import CharField
from django.db.models.signals import post_delete, post_save
from django.dispatch.dispatcher import receiver
from django.utils import timezone
from hdx.configuration import Configuration
from hdx_exports.hdx_export_set import HDXExportSet
from feature_selection.feature_selection import FeatureSelection
from utils import FORMAT_NAMES
HDX_URL_PREFIX = Configuration.create(
hdx_site=settings.HDX_SITE,
hdx_key=settings.HDX_API_KEY,
)
LOG = logging.getLogger(__name__)
# construct the upload path for export config files..
def get_upload_path(instance, filename):
"""
Construct the path to where the uploaded config file is to be stored.
"""
configtype = instance.config_type.lower()
# sanitize the filename here..
path = 'export/config/{0}/{1}'.format(configtype, instance.filename)
return path