def countriesdata(cls, use_live=True):
# type: (bool) -> List[Dict[Dict]]
"""
Read countries data from OCHA countries feed (falling back to file)
Args:
use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
Returns:
List[Dict[Dict]]: Countries dictionaries
"""
if cls._countriesdata is None:
countries = None
if use_live:
try:
countries = hxl.data(cls._ochaurl)
except IOError:
logger.exception(
'Download from OCHA feed failed! Falling back to stored file.'
)
if countries is None:
countries = hxl.data(script_dir_plus_file(
'Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv',
Country),
allow_local=True)
cls.set_countriesdata(countries)
return cls._countriesdata
def __init__(self, configuration: Configuration, initial_data: dict):
super(HDXObject, self).__init__(initial_data)
self.configuration = configuration
self.old_data = None
version_file = open(script_dir_plus_file(join('..', 'version.txt'), HDXObject))
version = version_file.read().strip()
self.hdxpostsite = ckanapi.RemoteCKAN(configuration.get_hdx_site_url(),
apikey=configuration.get_api_key(),
user_agent='HDXPythonLibrary/%s' % version)
def test_ocha_feed_file_working(self):
countries = hxl.data(script_dir_plus_file('Countries_UZB_Deleted.csv', TestCountry), allow_local=True)
Country.set_countriesdata(countries)
assert Country.get_iso3_country_code('UZBEKISTAN') is None
assert Country.get_iso3_country_code('south sudan') == 'SSD'
Country.set_ocha_url()
Country._countriesdata = None
assert Country.get_iso3_country_code('UZBEKISTAN', use_live=True) == 'UZB'
Country.set_ocha_url('NOTEXIST')
Country._countriesdata = None
assert Country.get_iso3_from_iso2('AF') == 'AFG'
def create_datastore_for_topline(self, delete_first: int = 0):
"""Creates a resource in the HDX datastore using the built in YAML definition for a topline
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
Returns:
None
"""
data = load_yaml(
script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'),
Resource))
self.create_datastore_from_dict_schema(data, delete_first)
def create_remoteckan(self):
# type: () -> ckanapi.RemoteCKAN
"""
Create remote CKAN instance from configuration
Returns:
ckanapi.RemoteCKAN: Remote CKAN instance
"""
version_file = open(script_dir_plus_file('version.txt', Configuration))
version = version_file.read().strip()
return ckanapi.RemoteCKAN(self.get_hdx_site_url(), apikey=self.get_api_key(),
user_agent='HDXPythonLibrary/%s' % version)
def create_datastore_for_topline(self, delete_first=0, path=None):
# type: (int, Optional[str]) -> None
"""For tabular data, create a resource in the HDX datastore which enables data preview in HDX using the built in
YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX.
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource))
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def main(hdx_key, user_agent, preprefix, hdx_site, db_url, save):
project_config_yaml = script_dir_plus_file('project_configuration.yml',
main)
site_url = Configuration.create(hdx_key=hdx_key,
hdx_site=hdx_site,
user_agent=user_agent,
preprefix=preprefix,
project_config_yaml=project_config_yaml)
logger.info('--------------------------------------------------')
logger.info('> HDX Site: %s' % site_url)
if db_url:
logger.info('> DB URL: %s' % db_url)
if 'postgres' in db_url:
result = urlparse(db_url)
username = result.username
password = result.password
database = result.path[1:]
hostname = result.hostname
connecting_string = 'Checking for PostgreSQL...'
while True:
try:
logger.info(connecting_string)
connection = psycopg2.connect(database=database,
user=username,
password=password,
host=hostname,
connect_timeout=3)
connection.close()
logger.info('PostgreSQL is running!')
break
except psycopg2.OperationalError:
time.sleep(1)
testsession = None
if save:
engine = create_engine('sqlite:///test_serialize.db',
poolclass=NullPool,
echo=False)
Session = sessionmaker(bind=engine)
TestBase.metadata.create_all(engine)
testsession = Session()
freshness = DataFreshness(db_url=db_url, testsession=testsession)
datasets_to_check, resources_to_check = freshness.process_datasets()
results, hash_results = freshness.check_urls(resources_to_check)
datasets_lastmodified = freshness.process_results(results, hash_results)
freshness.update_dataset_last_modified(datasets_to_check,
datasets_lastmodified)
freshness.output_counts()
freshness.close()
logger.info('Freshness completed!')
def create_datastore_for_topline(self, delete_first=0, path=None):
# type: (int, Optional[str]) -> None
"""For csvs, create a resource in the HDX datastore which enables data preview in HDX using the built in
YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX.
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource))
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create(**kwargs) -> 'Configuration':
"""
Create
Args:
path (str): Path to HDX key
Returns:
str: HDX api key
"""
hdx.configuration = Configuration(**kwargs)
version_file = open(script_dir_plus_file('version.txt', Configuration))
version = version_file.read().strip()
hdx.remoteckan = ckanapi.RemoteCKAN(
hdx.configuration.get_hdx_site_url(),
apikey=hdx.configuration.get_api_key(),
user_agent='HDXPythonLibrary/%s' % version)
return hdx.configuration
def create_remoteckan(
self,
session=get_session(method_whitelist=frozenset(
['HEAD', 'TRACE', 'GET', 'POST', 'PUT', 'OPTIONS', 'DELETE']))):
# type: () -> ckanapi.RemoteCKAN
"""
Create remote CKAN instance from configuration
Args:
session: requests Session object to use. Defaults to calling hdx.utilities.session.get_session()
Returns:
ckanapi.RemoteCKAN: Remote CKAN instance
"""
version_file = open(script_dir_plus_file('version.txt', Configuration))
version = version_file.read().strip()
return ckanapi.RemoteCKAN(self.get_hdx_site_url(),
session=session,
user_agent='HDXPythonLibrary/%s' % version)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration file given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration file given!')
else:
if not hdx_config_yaml:
logger.info('No HDX configuration parameter. Using default.')
hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml',
Configuration)
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration file given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration file given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_config_dict,
project_config_dict)
self.hdx_read_only = kwargs.get('hdx_read_only', False)
if not self.hdx_read_only:
if 'hdx_key' in kwargs:
self.data['api_key'] = kwargs.get('hdx_key')
else:
hdx_key_file = kwargs.get('hdx_key_file',
join(expanduser('~'), '.hdxkey'))
""" :type : str"""
self.data['api_key'] = self.load_api_key(hdx_key_file)
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test')
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)
def setup_logging(**kwargs) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get('smtp_config_dict', None)
if smtp_config_dict:
smtp_config_found = True
print('Loading smtp configuration customisations from dictionary')
smtp_config_json = kwargs.get('smtp_config_json', '')
if smtp_config_json:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_json)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get('smtp_config_yaml', '')
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_yaml)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get('logging_config_dict', None)
if logging_config_dict:
logging_config_found = True
print('Loading logging configuration from dictionary')
logging_config_json = kwargs.get('logging_config_json', '')
if logging_config_json:
if logging_config_found:
raise LoggingError(
'More than one logging configuration file given!')
logging_config_found = True
print('Loading logging configuration from: %s' % logging_config_json)
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get('logging_config_yaml', '')
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
'More than one logging configuration file given!')
else:
if not logging_config_yaml:
print('No logging configuration parameter. Using default.')
logging_config_yaml = script_dir_plus_file(
'logging_configuration.yml', setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
'logging_smtp_configuration.yml', setup_logging)
print('Loading base SMTP logging configuration from: %s' %
logging_smtp_config_yaml)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print('Loading logging configuration from: %s' % logging_config_yaml)
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict, logging_smtp_config_dict, smtp_config_dict
])
else:
raise LoggingError(
'SMTP logging configuration file given but not using default logging configuration!'
)
logging.config.dictConfig(logging_config_dict)
def setup_logging(**kwargs: Any) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get("smtp_config_dict", None)
if smtp_config_dict:
smtp_config_found = True
print("Loading smtp configuration customisations from dictionary")
smtp_config_json = kwargs.get("smtp_config_json", "")
if smtp_config_json:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_json}"
)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get("smtp_config_yaml", "")
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_yaml}"
)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get("logging_config_dict", None)
if logging_config_dict:
logging_config_found = True
print("Loading logging configuration from dictionary")
logging_config_json = kwargs.get("logging_config_json", "")
if logging_config_json:
if logging_config_found:
raise LoggingError(
"More than one logging configuration file given!")
logging_config_found = True
print(f"Loading logging configuration from: {logging_config_json}")
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get("logging_config_yaml", "")
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
"More than one logging configuration file given!")
else:
if not logging_config_yaml:
print("No logging configuration parameter. Using default.")
logging_config_yaml = script_dir_plus_file(
"logging_configuration.yml", setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
"logging_smtp_configuration.yml", setup_logging)
print(
f"Loading base SMTP logging configuration from: {logging_smtp_config_yaml}"
)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print(f"Loading logging configuration from: {logging_config_yaml}")
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict,
logging_smtp_config_dict,
smtp_config_dict,
])
else:
raise LoggingError(
"SMTP logging configuration file given but not using default logging configuration!"
)
file_only = os.getenv("LOG_FILE_ONLY")
if file_only is not None and file_only.lower() not in [
"false",
"f",
"n",
"no",
"0",
]:
root = logging_config_dict.get("root")
if root is not None:
handlers = root.get("handlers", list())
for i, handler in enumerate(handlers):
if handler.lower() == "console":
del handlers[i]
break
logging.config.dictConfig(logging_config_dict)
db_url = getenv("DB_URL")
if db_url and "://" not in db_url:
db_url = f"postgresql://{db_url}"
gsheet_auth = args.gsheet_auth
if gsheet_auth is None:
gsheet_auth = getenv("GSHEET_AUTH")
email_server = args.email_server
if email_server is None:
email_server = getenv("EMAIL_SERVER")
failure_emails = args.failure_emails
if failure_emails is None:
failure_emails = getenv("FAILURE_EMAILS")
sysadmin_emails = args.sysadmin_emails
if sysadmin_emails is None:
sysadmin_emails = getenv("SYSADMIN_EMAILS")
project_config_yaml = script_dir_plus_file("project_configuration.yml",
main)
facade(
main,
hdx_key=hdx_key,
user_agent=user_agent,
preprefix=preprefix,
hdx_site=hdx_site,
project_config_yaml=project_config_yaml,
db_url=db_url,
db_params=args.db_params,
gsheet_auth=gsheet_auth,
email_server=email_server,
failure_emails=failure_emails,
sysadmin_emails=sysadmin_emails,
email_test=args.email_test,
spreadsheet_test=args.spreadsheet_test,
def get_country_version():
version_file = open(script_dir_plus_file('version.txt', get_country_version))
return version_file.read().strip()
class Configuration(UserDict, object):
"""Configuration for HDX
Args:
**kwargs: See below
user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
hdx_config_json (str): Path to JSON HDX configuration OR
hdx_config_yaml (str): Path to YAML HDX configuration
project_config_dict (dict): Project configuration dictionary OR
project_config_json (str): Path to JSON Project configuration OR
project_config_yaml (str): Path to YAML Project configuration
hdx_base_config_dict (dict): HDX base configuration dictionary OR
hdx_base_config_json (str): Path to JSON HDX base configuration OR
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.
"""
_configuration = None
home_folder = expanduser('~')
default_hdx_base_config_yaml = script_dir_plus_file(
'hdx_base_configuration.yml', ConfigurationError)
default_hdx_config_yaml = join(home_folder, '.hdx_configuration.yml')
apiversion = get_api_version()
prefix = 'HDXPythonLibrary/%s' % apiversion
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
self._remoteckan = None
self._emailer = None
hdx_base_config_found = False
hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None)
if hdx_base_config_dict:
hdx_base_config_found = True
logger.info('Loading HDX base configuration from dictionary')
hdx_base_config_json = kwargs.get('hdx_base_config_json', '')
if hdx_base_config_json:
if hdx_base_config_found:
raise ConfigurationError(
'More than one HDX base configuration given!')
hdx_base_config_found = True
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_json)
hdx_base_config_dict = load_json(hdx_base_config_json)
hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '')
if hdx_base_config_found:
if hdx_base_config_yaml:
raise ConfigurationError(
'More than one HDX base configuration given!')
else:
if not hdx_base_config_yaml:
hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml
logger.info(
'No HDX base configuration parameter. Using default base configuration file: %s.'
% hdx_base_config_yaml)
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_yaml)
hdx_base_config_dict = load_yaml(hdx_base_config_yaml)
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration given!')
else:
if not hdx_config_yaml:
hdx_config_yaml = Configuration.default_hdx_config_yaml
if isfile(hdx_config_yaml):
logger.info(
'No HDX configuration parameter. Using default configuration file: %s.'
% hdx_config_yaml)
else:
logger.info(
'No HDX configuration parameter and no configuration file at default path: %s.'
% hdx_config_yaml)
hdx_config_yaml = None
hdx_config_dict = dict()
if hdx_config_yaml:
logger.info('Loading HDX configuration from: %s' %
hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
self.data = merge_two_dictionaries(hdx_base_config_dict,
hdx_config_dict)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_base_config_dict,
project_config_dict)
ua = kwargs.get('full_agent')
if ua:
self.user_agent = ua
else:
try:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**kwargs)
except UserAgentError:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**self.data)
self.hdx_read_only = kwargs.get('hdx_read_only',
self.data.get('hdx_read_only', False))
logger.info('Read only access to HDX: %s' % str(self.hdx_read_only))
self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key'))
if not self.hdx_key and not self.hdx_read_only:
raise ConfigurationError(
'No HDX API key supplied as a parameter or in configuration!')
hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url'))
if hdx_url:
self.hdx_site = 'hdx_custom_site'
hdx_url = hdx_url.rstrip('/')
self.data[self.hdx_site] = {'url': hdx_url}
else:
self.hdx_site = 'hdx_%s_site' % kwargs.get(
'hdx_site', self.data.get('hdx_site', 'test'))
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)
def set_read_only(self, read_only=True):
# type: (bool) -> None
"""
Set HDX read only flag
Args:
read_only (bool): Value to set HDX read only flag. Defaults to True.
Returns:
None
"""
self.hdx_read_only = read_only
def set_api_key(self, apikey):
# type: (str) -> None
"""
Set HDX api key
Args:
apikey (str): Value to set api key.
Returns:
None
"""
self.hdx_key = apikey
def get_api_key(self):
# type: () -> Optional[str]
"""
Return HDX api key or None if read only
Returns:
Optional[str]: HDX api key or None if read only
"""
if self.hdx_read_only:
return None
return self.hdx_key
def get_user_agent(self):
# type: () -> str
"""
Return user agent
Returns:
str: User agent
"""
return self.user_agent
def get_hdx_site_url(self):
# type: () -> str
"""
Return HDX web site url
Returns:
str: HDX web site url
"""
return self.data[self.hdx_site]['url']
def get_dataset_url(self, name):
# type: (str) -> str
"""
Return HDX dataset url given dataset name
Args:
name: Dataset name
Returns:
str: HDX dataset url
"""
return '%s/dataset/%s' % (self.get_hdx_site_url(), name)
def _get_credentials(self):
# type: () -> Optional[Tuple[str, str]]
"""
Return HDX site username and password
Returns:
Optional[Tuple[str, str]]: HDX site username and password or None
"""
site = self.data[self.hdx_site]
username = site.get('username')
if username:
return b64decode(username).decode('utf-8'), b64decode(
site['password']).decode('utf-8')
else:
return None
def remoteckan(self):
# type: () -> ckanapi.RemoteCKAN
"""
Return the remote CKAN object (see ckanapi library)
Returns:
ckanapi.RemoteCKAN: The remote CKAN object
"""
if self._remoteckan is None:
raise ConfigurationError(
'There is no remote CKAN set up! Use Configuration.create(**kwargs)'
)
return self._remoteckan
def call_remoteckan(self, *args, **kwargs):
# type: (Any, Any) -> Dict
"""
Calls the remote CKAN
Args:
*args: Arguments to pass to remote CKAN call_action method
**kwargs: Keyword arguments to pass to remote CKAN call_action method
Returns:
Dict: The response from the remote CKAN call_action method
"""
requests_kwargs = kwargs.get('requests_kwargs', dict())
credentials = self._get_credentials()
if credentials:
requests_kwargs['auth'] = credentials
kwargs['requests_kwargs'] = requests_kwargs
apikey = kwargs.get('apikey', self.get_api_key())
kwargs['apikey'] = apikey
return self.remoteckan().call_action(*args, **kwargs)
@staticmethod
def _environment_variables(**kwargs):
# type: (Any) -> Any
"""
Overwrite keyword arguments with environment variables
Args:
**kwargs: See below
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test. Defaults to test.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
Returns:
kwargs: Changed keyword arguments
"""
hdx_key = os.getenv('HDX_KEY')
if hdx_key is not None:
kwargs['hdx_key'] = hdx_key
hdx_url = os.getenv('HDX_URL')
if hdx_url is not None:
kwargs['hdx_url'] = hdx_url
else:
hdx_site = os.getenv('HDX_SITE')
if hdx_site is not None:
kwargs['hdx_site'] = hdx_site
return kwargs
@classmethod
def create_remoteckan(cls,
site_url,
user_agent=None,
user_agent_config_yaml=None,
user_agent_lookup=None,
use_env=False,
session=None,
**kwargs):
# type: (str, Optional[str], Optional[str], Optional[str], bool, requests.Session, Any) -> ckanapi.RemoteCKAN
"""
Create remote CKAN instance from configuration
Args:
site_url (str): Site url.
user_agent (Optional[str]): User agent string. HDXPythonLibrary/X.X.X- is prefixed.
user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied.
use_env (bool): Whether to read environment variables. Defaults to False.
session (requests.Session): requests Session object to use. Defaults to calling hdx.utilities.session.get_session()
Returns:
ckanapi.RemoteCKAN: Remote CKAN instance
"""
if not session:
whitelist = frozenset(
['HEAD', 'TRACE', 'GET', 'POST', 'PUT', 'OPTIONS', 'DELETE'])
session = get_session(user_agent,
user_agent_config_yaml,
user_agent_lookup,
use_env,
prefix=Configuration.prefix,
method_whitelist=whitelist,
**kwargs)
ua = session.headers['User-Agent']
else:
ua = kwargs.get('full_agent')
if not ua:
ua = UserAgent.get(user_agent,
user_agent_config_yaml,
user_agent_lookup,
prefix=Configuration.prefix,
**kwargs)
return ckanapi.RemoteCKAN(site_url, user_agent=ua, session=session)
def setup_remoteckan(self, remoteckan=None, **kwargs):
# type: (Optional[ckanapi.RemoteCKAN], Any) -> None
"""
Set up remote CKAN from provided CKAN or by creating from configuration
Args:
remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.
Returns:
None
"""
if remoteckan is None:
self._remoteckan = self.create_remoteckan(
self.get_hdx_site_url(),
full_agent=self.get_user_agent(),
**kwargs)
else:
self._remoteckan = remoteckan
def emailer(self):
# type: () -> Email
"""
Return the Email object (see :any:`Email`)
Returns:
Email: The email object
"""
if self._emailer is None:
raise ConfigurationError(
'There is no emailer set up! Use setup_emailer(Any)')
return self._emailer
def setup_emailer(self, **kwargs):
# type: (Any) -> None
"""
Set up emailer. Parameters in dictionary or file (eg. yaml below):
| connection_type: "ssl" ("ssl" for smtp ssl or "lmtp", otherwise basic smtp is assumed)
| host: "localhost"
| port: 123
| local_hostname: "mycomputer.fqdn.com"
| timeout: 3
| username: "******"
| password: "******"
Args:
**kwargs: See below
email_config_dict (dict): HDX configuration dictionary OR
email_config_json (str): Path to JSON HDX configuration OR
email_config_yaml (str): Path to YAML HDX configuration. Defaults to ~/hdx_email_configuration.yml.
Returns:
None
"""
self._emailer = Email(**kwargs)
@classmethod
def read(cls):
# type: () -> 'Configuration'
"""
Read the HDX configuration
Returns:
Configuration: The HDX configuration
"""
if cls._configuration is None:
raise ConfigurationError(
'There is no HDX configuration! Use Configuration.create(**kwargs)'
)
return cls._configuration
@classmethod
def setup(cls, configuration=None, **kwargs):
# type: (Optional['Configuration'], Any) -> None
"""
Set up the HDX configuration
Args:
configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
**kwargs: See below
user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
hdx_config_json (str): Path to JSON HDX configuration OR
hdx_config_yaml (str): Path to YAML HDX configuration
project_config_dict (dict): Project configuration dictionary OR
project_config_json (str): Path to JSON Project configuration OR
project_config_yaml (str): Path to YAML Project configuration
hdx_base_config_dict (dict): HDX base configuration dictionary OR
hdx_base_config_json (str): Path to JSON HDX base configuration OR
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.
Returns:
None
"""
if configuration is None:
cls._configuration = Configuration(**kwargs)
else:
cls._configuration = configuration
@classmethod
def _create(cls, configuration=None, remoteckan=None, **kwargs):
# type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str
"""
Create HDX configuration
Args:
configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.
**kwargs: See below
user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
hdx_config_json (str): Path to JSON HDX configuration OR
hdx_config_yaml (str): Path to YAML HDX configuration
project_config_dict (dict): Project configuration dictionary OR
project_config_json (str): Path to JSON Project configuration OR
project_config_yaml (str): Path to YAML Project configuration
hdx_base_config_dict (dict): HDX base configuration dictionary OR
hdx_base_config_json (str): Path to JSON HDX base configuration OR
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.
Returns:
str: HDX site url
"""
kwargs = cls._environment_variables(**kwargs)
cls.setup(configuration, **kwargs)
cls._configuration.setup_remoteckan(remoteckan, **kwargs)
return cls._configuration.get_hdx_site_url()
@classmethod
def create(cls, configuration=None, remoteckan=None, **kwargs):
# type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str
"""
Create HDX configuration. Can only be called once (will raise an error if called more than once).
Args:
configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.
**kwargs: See below
user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
hdx_config_json (str): Path to JSON HDX configuration OR
hdx_config_yaml (str): Path to YAML HDX configuration
project_config_dict (dict): Project configuration dictionary OR
project_config_json (str): Path to JSON Project configuration OR
project_config_yaml (str): Path to YAML Project configuration
hdx_base_config_dict (dict): HDX base configuration dictionary OR
hdx_base_config_json (str): Path to JSON HDX base configuration OR
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.
Returns:
str: HDX site url
"""
if cls._configuration is not None:
raise ConfigurationError('Configuration already created!')
return cls._create(configuration=configuration,
remoteckan=remoteckan,
**kwargs)
@classmethod
def delete(cls):
# type: () -> None
"""
Delete global HDX configuration.
Returns:
None
"""
cls._configuration = None
def get_api_version():
version_file = open(script_dir_plus_file('version.txt', get_api_version))
return version_file.read().strip()
