def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}]) # add locations used in tests
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'commodities'
}, {
'name': 'prices'
}, {
'name': 'markets'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'indicators'
}, {
'name': 'health'
}, {
'name': 'education'
}, {
'name': 'socioeconomic'
}, {
'name': 'demographics'
}, {
'name': 'development'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
return Configuration.read()
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}, {
'name': 'tza',
'title': 'Tanzania'
}, {
'name': 'world',
'title': 'World'
}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'violence and conflict'
}, {
'name': 'displacement'
}, {
'name': 'internally displaced persons - idp'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
def test_set_hdx_key_value(self, empty_hdx_key_file, project_config_yaml):
with pytest.raises(LoadError):
Configuration.load_api_key(empty_hdx_key_file)
Configuration._create(hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
configuration = Configuration.read()
assert configuration.get_api_key() == 'TEST_HDX_KEY'
configuration.set_api_key('NEW API KEY')
assert configuration.get_api_key() == 'NEW API KEY'
Configuration._create(hdx_site='prod',
hdx_read_only=True,
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert Configuration.read().get_api_key() is None
configuration = Configuration.read()
configuration.set_api_key('TEST API KEY')
assert configuration.get_api_key() is None
configuration.set_read_only(False)
assert configuration.get_api_key() == 'TEST API KEY'
configuration.set_read_only(True)
assert configuration.get_api_key() is None
configuration.set_api_key('NEW API KEY')
configuration.set_read_only(False)
assert configuration.get_api_key() == 'NEW API KEY'
def configuration(self):
Configuration._create(hdx_read_only=True, user_agent='test',
project_config_yaml=join('tests', 'config', 'project_configuration.yml'))
Locations.set_validlocations([{'name': 'gin', 'title': 'Guinea'}]) # add locations used in tests
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {'tags': [{'name': 'hxl'}, {'name': 'food security'}, {'name': 'indicators'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'}
def test_remoteckan_validlocations(self, project_config_yaml):
Configuration._create(hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
remoteckan = ckanapi.RemoteCKAN('http://lalala',
apikey='12345',
user_agent='HDXPythonLibrary/1.0')
Configuration.read().setup_remoteckan(remoteckan)
assert Configuration.read().remoteckan() == remoteckan
remoteckan = ckanapi.RemoteCKAN('http://hahaha',
apikey='54321',
user_agent='HDXPythonLibrary/0.5')
Configuration._create(remoteckan=remoteckan,
hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan() == remoteckan
Configuration.read()._remoteckan = None
with pytest.raises(ConfigurationError):
Configuration.read().remoteckan()
Configuration.delete()
with pytest.raises(ConfigurationError):
Configuration.read().remoteckan()
def test_hdx_configuration_yaml(self, hdx_config_yaml, hdx_base_config_yaml, project_config_yaml):
Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml,
hdx_base_config_yaml=hdx_base_config_yaml, project_config_yaml=project_config_yaml)
expected_configuration = {
'hdx_site': 'prod',
'hdx_read_only': False,
'hdx_key': '12345',
'hdx_prod_site': {
'url': 'https://data.humdata.org',
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org',
'username': '******',
'password': '******'
},
'dataset': {'required_fields': [
'name',
'title',
'dataset_date',
]},
'resource': {'required_fields': ['package_id', 'name', 'description']},
'showcase': {'required_fields': ['name', 'title']},
'approved_tags_vocabulary': 'Topics',
'tags_list_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Accepted_Tags.csv',
'tags_mapping_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Tag_Mapping.csv',
}
assert Configuration.read() == expected_configuration
def configuration(self):
Configuration._create(hdx_read_only=True, user_agent='test',
project_config_yaml=join('tests', 'config', 'project_configuration.yml'))
Locations.set_validlocations([{'name': 'arg', 'title': 'Argentina'}]) # add locations used in tests
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {'tags': [{'name': 'sustainable development'}, {'name': 'demographics'}, {'name': 'socioeconomics'}, {'name': 'education'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'}
def configuration(self):
Configuration._create(user_agent='test',
hdx_key='12345',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}, {
'name': 'cmr',
'title': 'Cameroon'
}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'violence and conflict'
}, {
'name': 'protests'
}, {
'name': 'security incidents'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
def test_hdx_configuration_yaml(self, hdx_key_file, hdx_config_yaml,
project_config_yaml):
Configuration._create(hdx_key_file=hdx_key_file,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org/',
'username': '******',
'password': '******'
},
'dataset': {
'required_fields': [
'name',
'title',
'dataset_date',
]
},
'resource': {
'required_fields': ['package_id', 'name', 'description']
},
'showcase': {
'required_fields': ['name', 'title']
},
}
assert Configuration.read() == expected_configuration
def configuration(self):
Configuration._create(
user_agent="test",
hdx_key="12345",
project_config_yaml=join("tests", "config",
"project_configuration.yml"),
)
Locations.set_validlocations([{"name": "bgd", "title": "Bangladesh"}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
"tags": [
{
"name": "hxl"
},
{
"name": "violence and conflict"
},
{
"name": "protests"
},
{
"name": "security incidents"
},
],
"id":
"4e61d464-4943-4e97-973a-84673c1aaa87",
"name":
"approved",
}
def test_get_hdx_key_site(self, hdx_config_yaml, project_config_yaml):
Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml,
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
actual_configuration = Configuration.read()
assert actual_configuration.get_api_key() == '12345'
assert actual_configuration.get_hdx_site_url() == 'https://data.humdata.org'
assert actual_configuration._get_credentials() is None
assert actual_configuration.get_dataset_url('mydataset') == 'https://data.humdata.org/dataset/mydataset'
def configuration(self):
Configuration._create(hdx_site='feature', user_agent='test', hdx_key='12345',
project_config_yaml=join('tests', 'config', 'project_configuration.yml'))
Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}, {'name': 'cmr', 'title': 'Cameroon'}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {'tags': [{'name': 'hxl'}, {'name': 'health'}, {'name': 'demographics'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'}
return Configuration.read()
def configuration(self):
Configuration._create(hdx_read_only=True,
hdx_site='prod',
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{'name': 'cog', 'title': 'Congo'}])
return Configuration.read()
def test_get_hdx_key_site(self, hdx_key_file, project_config_yaml):
Configuration._create(hdx_site='prod',
hdx_key_file=hdx_key_file,
hdx_config_dict={},
project_config_yaml=project_config_yaml)
actual_configuration = Configuration.read()
assert actual_configuration.get_api_key() == '12345'
assert actual_configuration.get_hdx_site_url(
) == 'https://data.humdata.org/'
assert actual_configuration._get_credentials() == ('', '')
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}]) # add locations used in tests
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'arg',
'title': 'Argentina'
}]) # add locations used in tests
Country.countriesdata(use_live=False)
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}, {
'name': 'pse',
'title': 'State of Palestine'
}])
Country.countriesdata(use_live=False)
def configuration(self):
Configuration._create(hdx_read_only=True,
hdx_site='prod',
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'afg',
'title': 'Afghanistan'
}, {
'name': 'pse',
'title': 'State of Palestine'
}])
return Configuration.read()
def facade(projectmainfn, **kwargs):
# type: (Callable[[], None], Any) -> bool
"""Facade that handles ScraperWiki and calls project main function
Args:
projectmainfn (() -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
bool: True = success, False = failure
"""
try:
#
# Setting up configuration
#
site_url = Configuration._create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> HDX Site: %s' % site_url)
projectmainfn()
except Exception as e:
logger.critical(e, exc_info=True)
scraperwiki.status('error', 'Run failed: %s' % sys.exc_info()[0])
return False
logger.info('Run completed successfully.\n')
scraperwiki.status('ok')
return True
def test_create_set_configuration(self, project_config_yaml):
Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
with pytest.raises(ConfigurationError):
Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
configuration = Configuration(user_agent='test', hdx_site='test', hdx_key='OTHER_TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
Configuration.setup(configuration)
assert Configuration.read() == configuration
Configuration.delete()
with pytest.raises(ConfigurationError):
Configuration.read()
Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().get_api_key() == 'TEST_HDX_KEY'
def test_set_hdx_key_value(self, project_config_yaml):
Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
configuration = Configuration.read()
assert configuration.get_api_key() == 'TEST_HDX_KEY'
configuration.set_api_key('NEW API KEY')
assert configuration.get_api_key() == 'NEW API KEY'
Configuration._create(user_agent='test', hdx_site='prod', hdx_read_only=True,
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().get_api_key() is None
configuration = Configuration.read()
configuration.set_api_key('TEST API KEY')
assert configuration.get_api_key() is None
configuration.set_read_only(False)
assert configuration.get_api_key() == 'TEST API KEY'
configuration.set_read_only(True)
assert configuration.get_api_key() is None
configuration.set_api_key('NEW API KEY')
configuration.set_read_only(False)
assert configuration.get_api_key() == 'NEW API KEY'
def configuration():
Configuration._create(hdx_read_only=True, user_agent='test')
Locations.set_validlocations([{'name': 'world', 'title': 'World'}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'energy'
}, {
'name': 'refugees'
}, {
'name': 'internally displaced persons - idp'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
def facade(projectmainfn, **kwargs):
# (Callable[[None], None], Any) -> None
"""Facade to simplify project setup that calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
None
"""
#
# Setting up configuration
#
site_url = Configuration._create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> HDX Site: %s' % site_url)
projectmainfn()
def facade(projectmainfn, **kwargs):
# (Callable[[None], None], Any) -> None
"""Facade to simplify project setup that calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
None
"""
#
# Setting up configuration
#
site_url = Configuration._create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> Using HDX Python API Library %s' % Configuration.apiversion)
logger.info('> HDX Site: %s' % site_url)
UserAgent.user_agent = Configuration.read().user_agent
projectmainfn()
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{
'name': 'mmr',
'title': 'Myanmar'
}, {
'name': 'sdn',
'title': 'Sudan'
}, {
'name': 'alb',
'title': 'Albania'
}, {
'name': 'yem',
'title': 'Yemen'
}]) # add locations used in tests
Country.countriesdata(False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'geodata'
}, {
'name': 'populated places - settlements'
}, {
'name': 'land use and land cover'
}, {
'name': 'erosion'
}, {
'name': 'landslides - mudslides'
}, {
'name': 'floods - storm surges'
}, {
'name': 'droughts'
}, {
'name': 'food assistance'
}, {
'name': 'hazards and risk'
}, {
'name': 'administrative divisions'
}, {
'name': 'food security'
}, {
'name': 'security'
}, {
'name': 'displaced persons locations - camps - shelters'
}, {
'name': 'refugees'
}, {
'name': 'internally displaced persons - idp'
}, {
'name': 'malnutrition'
}, {
'name': 'nutrition'
}, {
'name': 'food assistance'
}, {
'name': 'roads'
}, {
'name': 'transportation'
}, {
'name': 'aviation'
}, {
'name': 'facilities and infrastructure'
}, {
'name': 'bridges'
}, {
'name': 'transportation'
}, {
'name': 'facilities and infrastructure'
}, {
'name': 'cold waves'
}, {
'name': 'cash assistance'
}, {
'name': 'acronyms'
}, {
'name': 'common operational dataset - cod'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
def configuration():
project_config_yaml = join('src', 'hdx', 'freshness', 'project_configuration.yml')
Configuration._create(hdx_site='prod', user_agent='test', hdx_read_only=True,
project_config_yaml=project_config_yaml)
def test_hdx_configuration_dict(self, hdx_key_file, project_config_yaml,
mocksmtp):
Configuration._create(hdx_site='prod',
hdx_key_file=hdx_key_file,
hdx_config_dict={
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'XYZ': {
'567': 987
}
},
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'XYZ': {
'567': 987
}
}
configuration = Configuration.read()
assert configuration == expected_configuration
smtp_initargs = {
'host': 'localhost',
'port': 123,
'local_hostname': 'mycomputer.fqdn.com',
'timeout': 3,
'source_address': ('machine', 456),
}
username = '******'
password = '******'
email_config_dict = {
'connection_type': 'ssl',
'username': username,
'password': password
}
email_config_dict.update(smtp_initargs)
recipients = ['*****@*****.**', '*****@*****.**', '*****@*****.**']
subject = 'hello'
body = 'hello there'
sender = '*****@*****.**'
mail_options = ['a', 'b']
rcpt_options = [1, 2]
with pytest.raises(ConfigurationError):
configuration.emailer()
configuration.setup_emailer(email_config_dict=email_config_dict)
email = configuration.emailer()
email.send(recipients,
subject,
body,
sender=sender,
mail_options=mail_options,
rcpt_options=rcpt_options)
assert email.server.type == 'smtpssl'
assert email.server.initargs == smtp_initargs
assert email.server.username == username
assert email.server.password == password
assert email.server.sender == sender
assert email.server.recipients == recipients
assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: hello
From: [email protected]
To: [email protected], [email protected], [email protected]
hello there'''
assert email.server.send_args == {
'mail_options': ['a', 'b'],
'rcpt_options': [1, 2]
}
def configuration(self):
Configuration._create(user_agent='test', hdx_key='12345',
project_config_yaml=join('tests', 'config', 'project_configuration.yml'))
Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}, {'name': 'cmr', 'title': 'Cameroon'}])
Country.countriesdata(use_live=False)
def test_project_configuration_yaml(self, hdx_key_file,
project_config_yaml):
Configuration._create(hdx_key_file=hdx_key_file,
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'hdx_demo_site': {
'url': 'https://demo-data.humdata.org/',
'username': '******',
'password': '******'
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org/',
'username': '******',
'password': '******'
},
'hdx_feature_site': {
'url': 'https://feature-data.humdata.org/',
'username': '******',
'password': '******'
},
'dataset': {
'required_fields': [
'name', 'private', 'title', 'notes', 'dataset_source',
'owner_org', 'maintainer', 'dataset_date',
'data_update_frequency', 'groups', 'license_id',
'methodology', 'tags'
]
},
'dataset-requestable': {
'required_fields': [
'name', 'title', 'notes', 'dataset_source', 'owner_org',
'maintainer', 'dataset_date', 'data_update_frequency',
'groups', 'tags', 'field_names', 'file_types',
'num_of_rows'
]
},
'resource': {
'required_fields': [
'package_id', 'name', 'format', 'url', 'description',
'url_type', 'resource_type'
]
},
'showcase': {
'required_fields':
['name', 'title', 'notes', 'url', 'image_url', 'tags']
},
'user': {
'required_fields': [
'name',
'email',
'password',
'fullname',
'about',
],
'ignore_on_update':
'password'
},
'organization': {
'required_fields': [
'name',
'title',
'description',
]
},
}
assert Configuration.read() == expected_configuration
def configuration(hdx_config_yaml, project_config_yaml):
Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml)
def configuration(self):
Configuration._create(user_agent='test',
hdx_read_only=True,
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
