def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }]) # add locations used in tests Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'commodities' }, { 'name': 'prices' }, { 'name': 'markets' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' }
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'indicators' }, { 'name': 'health' }, { 'name': 'education' }, { 'name': 'socioeconomic' }, { 'name': 'demographics' }, { 'name': 'development' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' } return Configuration.read()
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }, { 'name': 'tza', 'title': 'Tanzania' }, { 'name': 'world', 'title': 'World' }]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'violence and conflict' }, { 'name': 'displacement' }, { 'name': 'internally displaced persons - idp' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' }
def test_set_hdx_key_value(self, empty_hdx_key_file, project_config_yaml): with pytest.raises(LoadError): Configuration.load_api_key(empty_hdx_key_file) Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) configuration = Configuration.read() assert configuration.get_api_key() == 'TEST_HDX_KEY' configuration.set_api_key('NEW API KEY') assert configuration.get_api_key() == 'NEW API KEY' Configuration._create(hdx_site='prod', hdx_read_only=True, hdx_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().get_api_key() is None configuration = Configuration.read() configuration.set_api_key('TEST API KEY') assert configuration.get_api_key() is None configuration.set_read_only(False) assert configuration.get_api_key() == 'TEST API KEY' configuration.set_read_only(True) assert configuration.get_api_key() is None configuration.set_api_key('NEW API KEY') configuration.set_read_only(False) assert configuration.get_api_key() == 'NEW API KEY'
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join('tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'gin', 'title': 'Guinea'}]) # add locations used in tests Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = {'tags': [{'name': 'hxl'}, {'name': 'food security'}, {'name': 'indicators'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'}
def test_remoteckan_validlocations(self, project_config_yaml): Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) remoteckan = ckanapi.RemoteCKAN('http://lalala', apikey='12345', user_agent='HDXPythonLibrary/1.0') Configuration.read().setup_remoteckan(remoteckan) assert Configuration.read().remoteckan() == remoteckan remoteckan = ckanapi.RemoteCKAN('http://hahaha', apikey='54321', user_agent='HDXPythonLibrary/0.5') Configuration._create(remoteckan=remoteckan, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan() == remoteckan Configuration.read()._remoteckan = None with pytest.raises(ConfigurationError): Configuration.read().remoteckan() Configuration.delete() with pytest.raises(ConfigurationError): Configuration.read().remoteckan()
def test_hdx_configuration_yaml(self, hdx_config_yaml, hdx_base_config_yaml, project_config_yaml): Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml, hdx_base_config_yaml=hdx_base_config_yaml, project_config_yaml=project_config_yaml) expected_configuration = { 'hdx_site': 'prod', 'hdx_read_only': False, 'hdx_key': '12345', 'hdx_prod_site': { 'url': 'https://data.humdata.org', }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org', 'username': '******', 'password': '******' }, 'dataset': {'required_fields': [ 'name', 'title', 'dataset_date', ]}, 'resource': {'required_fields': ['package_id', 'name', 'description']}, 'showcase': {'required_fields': ['name', 'title']}, 'approved_tags_vocabulary': 'Topics', 'tags_list_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Accepted_Tags.csv', 'tags_mapping_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Tag_Mapping.csv', } assert Configuration.read() == expected_configuration
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join('tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'arg', 'title': 'Argentina'}]) # add locations used in tests Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = {'tags': [{'name': 'sustainable development'}, {'name': 'demographics'}, {'name': 'socioeconomics'}, {'name': 'education'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'}
def configuration(self): Configuration._create(user_agent='test', hdx_key='12345', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }, { 'name': 'cmr', 'title': 'Cameroon' }]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'violence and conflict' }, { 'name': 'protests' }, { 'name': 'security incidents' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' }
def test_hdx_configuration_yaml(self, hdx_key_file, hdx_config_yaml, project_config_yaml): Configuration._create(hdx_key_file=hdx_key_file, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org/', 'username': '******', 'password': '******' }, 'dataset': { 'required_fields': [ 'name', 'title', 'dataset_date', ] }, 'resource': { 'required_fields': ['package_id', 'name', 'description'] }, 'showcase': { 'required_fields': ['name', 'title'] }, } assert Configuration.read() == expected_configuration
def configuration(self): Configuration._create( user_agent="test", hdx_key="12345", project_config_yaml=join("tests", "config", "project_configuration.yml"), ) Locations.set_validlocations([{"name": "bgd", "title": "Bangladesh"}]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { "tags": [ { "name": "hxl" }, { "name": "violence and conflict" }, { "name": "protests" }, { "name": "security incidents" }, ], "id": "4e61d464-4943-4e97-973a-84673c1aaa87", "name": "approved", }
def test_get_hdx_key_site(self, hdx_config_yaml, project_config_yaml): Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml, hdx_base_config_dict={}, project_config_yaml=project_config_yaml) actual_configuration = Configuration.read() assert actual_configuration.get_api_key() == '12345' assert actual_configuration.get_hdx_site_url() == 'https://data.humdata.org' assert actual_configuration._get_credentials() is None assert actual_configuration.get_dataset_url('mydataset') == 'https://data.humdata.org/dataset/mydataset'
def configuration(self): Configuration._create(hdx_site='feature', user_agent='test', hdx_key='12345', project_config_yaml=join('tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}, {'name': 'cmr', 'title': 'Cameroon'}]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = {'tags': [{'name': 'hxl'}, {'name': 'health'}, {'name': 'demographics'}], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved'} return Configuration.read()
def configuration(self): Configuration._create(hdx_read_only=True, hdx_site='prod', user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'cog', 'title': 'Congo'}]) return Configuration.read()
def test_get_hdx_key_site(self, hdx_key_file, project_config_yaml): Configuration._create(hdx_site='prod', hdx_key_file=hdx_key_file, hdx_config_dict={}, project_config_yaml=project_config_yaml) actual_configuration = Configuration.read() assert actual_configuration.get_api_key() == '12345' assert actual_configuration.get_hdx_site_url( ) == 'https://data.humdata.org/' assert actual_configuration._get_credentials() == ('', '')
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }]) # add locations used in tests
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'arg', 'title': 'Argentina' }]) # add locations used in tests Country.countriesdata(use_live=False)
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }, { 'name': 'pse', 'title': 'State of Palestine' }]) Country.countriesdata(use_live=False)
def configuration(self): Configuration._create(hdx_read_only=True, hdx_site='prod', user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'afg', 'title': 'Afghanistan' }, { 'name': 'pse', 'title': 'State of Palestine' }]) return Configuration.read()
def facade(projectmainfn, **kwargs): # type: (Callable[[], None], Any) -> bool """Facade that handles ScraperWiki and calls project main function Args: projectmainfn (() -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: bool: True = success, False = failure """ try: # # Setting up configuration # site_url = Configuration._create(**kwargs) logger.info('--------------------------------------------------') logger.info('> HDX Site: %s' % site_url) projectmainfn() except Exception as e: logger.critical(e, exc_info=True) scraperwiki.status('error', 'Run failed: %s' % sys.exc_info()[0]) return False logger.info('Run completed successfully.\n') scraperwiki.status('ok') return True
def test_create_set_configuration(self, project_config_yaml): Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) with pytest.raises(ConfigurationError): Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) configuration = Configuration(user_agent='test', hdx_site='test', hdx_key='OTHER_TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) Configuration.setup(configuration) assert Configuration.read() == configuration Configuration.delete() with pytest.raises(ConfigurationError): Configuration.read() Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().get_api_key() == 'TEST_HDX_KEY'
def test_set_hdx_key_value(self, project_config_yaml): Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) configuration = Configuration.read() assert configuration.get_api_key() == 'TEST_HDX_KEY' configuration.set_api_key('NEW API KEY') assert configuration.get_api_key() == 'NEW API KEY' Configuration._create(user_agent='test', hdx_site='prod', hdx_read_only=True, hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().get_api_key() is None configuration = Configuration.read() configuration.set_api_key('TEST API KEY') assert configuration.get_api_key() is None configuration.set_read_only(False) assert configuration.get_api_key() == 'TEST API KEY' configuration.set_read_only(True) assert configuration.get_api_key() is None configuration.set_api_key('NEW API KEY') configuration.set_read_only(False) assert configuration.get_api_key() == 'NEW API KEY'
def configuration(): Configuration._create(hdx_read_only=True, user_agent='test') Locations.set_validlocations([{'name': 'world', 'title': 'World'}]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'energy' }, { 'name': 'refugees' }, { 'name': 'internally displaced persons - idp' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' }
def facade(projectmainfn, **kwargs): # (Callable[[None], None], Any) -> None """Facade to simplify project setup that calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: None """ # # Setting up configuration # site_url = Configuration._create(**kwargs) logger.info('--------------------------------------------------') logger.info('> HDX Site: %s' % site_url) projectmainfn()
def facade(projectmainfn, **kwargs): # (Callable[[None], None], Any) -> None """Facade to simplify project setup that calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: None """ # # Setting up configuration # site_url = Configuration._create(**kwargs) logger.info('--------------------------------------------------') logger.info('> Using HDX Python API Library %s' % Configuration.apiversion) logger.info('> HDX Site: %s' % site_url) UserAgent.user_agent = Configuration.read().user_agent projectmainfn()
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{ 'name': 'mmr', 'title': 'Myanmar' }, { 'name': 'sdn', 'title': 'Sudan' }, { 'name': 'alb', 'title': 'Albania' }, { 'name': 'yem', 'title': 'Yemen' }]) # add locations used in tests Country.countriesdata(False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'geodata' }, { 'name': 'populated places - settlements' }, { 'name': 'land use and land cover' }, { 'name': 'erosion' }, { 'name': 'landslides - mudslides' }, { 'name': 'floods - storm surges' }, { 'name': 'droughts' }, { 'name': 'food assistance' }, { 'name': 'hazards and risk' }, { 'name': 'administrative divisions' }, { 'name': 'food security' }, { 'name': 'security' }, { 'name': 'displaced persons locations - camps - shelters' }, { 'name': 'refugees' }, { 'name': 'internally displaced persons - idp' }, { 'name': 'malnutrition' }, { 'name': 'nutrition' }, { 'name': 'food assistance' }, { 'name': 'roads' }, { 'name': 'transportation' }, { 'name': 'aviation' }, { 'name': 'facilities and infrastructure' }, { 'name': 'bridges' }, { 'name': 'transportation' }, { 'name': 'facilities and infrastructure' }, { 'name': 'cold waves' }, { 'name': 'cash assistance' }, { 'name': 'acronyms' }, { 'name': 'common operational dataset - cod' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' }
def configuration(): project_config_yaml = join('src', 'hdx', 'freshness', 'project_configuration.yml') Configuration._create(hdx_site='prod', user_agent='test', hdx_read_only=True, project_config_yaml=project_config_yaml)
def test_hdx_configuration_dict(self, hdx_key_file, project_config_yaml, mocksmtp): Configuration._create(hdx_site='prod', hdx_key_file=hdx_key_file, hdx_config_dict={ 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'XYZ': { '567': 987 } }, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'XYZ': { '567': 987 } } configuration = Configuration.read() assert configuration == expected_configuration smtp_initargs = { 'host': 'localhost', 'port': 123, 'local_hostname': 'mycomputer.fqdn.com', 'timeout': 3, 'source_address': ('machine', 456), } username = '******' password = '******' email_config_dict = { 'connection_type': 'ssl', 'username': username, 'password': password } email_config_dict.update(smtp_initargs) recipients = ['*****@*****.**', '*****@*****.**', '*****@*****.**'] subject = 'hello' body = 'hello there' sender = '*****@*****.**' mail_options = ['a', 'b'] rcpt_options = [1, 2] with pytest.raises(ConfigurationError): configuration.emailer() configuration.setup_emailer(email_config_dict=email_config_dict) email = configuration.emailer() email.send(recipients, subject, body, sender=sender, mail_options=mail_options, rcpt_options=rcpt_options) assert email.server.type == 'smtpssl' assert email.server.initargs == smtp_initargs assert email.server.username == username assert email.server.password == password assert email.server.sender == sender assert email.server.recipients == recipients assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: hello From: [email protected] To: [email protected], [email protected], [email protected] hello there''' assert email.server.send_args == { 'mail_options': ['a', 'b'], 'rcpt_options': [1, 2] }
def configuration(self): Configuration._create(user_agent='test', hdx_key='12345', project_config_yaml=join('tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}, {'name': 'cmr', 'title': 'Cameroon'}]) Country.countriesdata(use_live=False)
def test_project_configuration_yaml(self, hdx_key_file, project_config_yaml): Configuration._create(hdx_key_file=hdx_key_file, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'hdx_demo_site': { 'url': 'https://demo-data.humdata.org/', 'username': '******', 'password': '******' }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org/', 'username': '******', 'password': '******' }, 'hdx_feature_site': { 'url': 'https://feature-data.humdata.org/', 'username': '******', 'password': '******' }, 'dataset': { 'required_fields': [ 'name', 'private', 'title', 'notes', 'dataset_source', 'owner_org', 'maintainer', 'dataset_date', 'data_update_frequency', 'groups', 'license_id', 'methodology', 'tags' ] }, 'dataset-requestable': { 'required_fields': [ 'name', 'title', 'notes', 'dataset_source', 'owner_org', 'maintainer', 'dataset_date', 'data_update_frequency', 'groups', 'tags', 'field_names', 'file_types', 'num_of_rows' ] }, 'resource': { 'required_fields': [ 'package_id', 'name', 'format', 'url', 'description', 'url_type', 'resource_type' ] }, 'showcase': { 'required_fields': ['name', 'title', 'notes', 'url', 'image_url', 'tags'] }, 'user': { 'required_fields': [ 'name', 'email', 'password', 'fullname', 'about', ], 'ignore_on_update': 'password' }, 'organization': { 'required_fields': [ 'name', 'title', 'description', ] }, } assert Configuration.read() == expected_configuration
def configuration(hdx_config_yaml, project_config_yaml): Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml)
def configuration(self): Configuration._create(user_agent='test', hdx_read_only=True, project_config_yaml=join( 'tests', 'config', 'project_configuration.yml'))