def test_load_empty(self, fixturesfolder):
loaderfolder = join(fixturesfolder, "loader")
with pytest.raises(LoadError):
load_file_to_str(join(loaderfolder, "empty.yml"))
with pytest.raises(LoadError):
load_yaml(join(loaderfolder, "empty.yml"))
with pytest.raises(LoadError):
load_json(join(loaderfolder, "empty.json"))
def retrieve_yaml(
self, url, filename, logstr=None, fallback=False, **kwargs
):
"""Retrieve YAML
Args:
url (str): URL to download
filename (str): Filename to use for saved file
logstr (Optional[str]): Text to use in log string to describe download. Defaults to filename.
fallback (bool): Whether to use static fallback if download fails. Defaults to False.
**kwargs: Parameters to pass to download call
Returns:
Union[Dict,List]: The data from the YAML file
"""
if not logstr:
logstr = filename
saved_path = join(self.saved_dir, filename)
if self.use_saved:
logger.info(f"Using saved {logstr} in {saved_path}")
ryaml = load_yaml(saved_path)
else:
try:
logger.info(
f"Downloading {logstr} from {self.get_url_logstr(url)}"
)
self.downloader.download(url, **kwargs)
ryaml = self.downloader.get_yaml()
if self.save:
logger.info(f"Saving {logstr} in {saved_path}")
save_yaml(ryaml, saved_path)
except DownloadError:
if not fallback:
raise
fallback_path = join(self.fallback_dir, filename)
logger.exception(
f"{logstr} download failed, using static data {fallback_path}!"
)
ryaml = load_yaml(fallback_path)
return ryaml
def create_datastore_for_topline(self, delete_first: int = 0):
"""Creates a resource in the HDX datastore using the built in YAML definition for a topline
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
Returns:
None
"""
data = load_yaml(
script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'),
Resource))
self.create_datastore_from_dict_schema(data, delete_first)
def create_datastore_for_topline(self, delete_first=0, path=None):
# type: (int, Optional[str]) -> None
"""For tabular data, create a resource in the HDX datastore which enables data preview in HDX using the built in
YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX.
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource))
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create_datastore_for_topline(self, delete_first=0, path=None):
# type: (int, Optional[str]) -> None
"""For csvs, create a resource in the HDX datastore which enables data preview in HDX using the built in
YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX.
Args:
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource))
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create_datastore_from_yaml_schema(self,
path: str,
delete_first: int = 0) -> None:
"""Creates a resource in the HDX datastore from a YAML file containing a list of fields and types of
form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key
Args:
path (str): Path to YAML file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'}
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
Returns:
None
"""
data = load_yaml(path)
self.create_datastore_from_dict_schema(data, delete_first)
def create_datastore_from_yaml_schema(self, yaml_path, delete_first=0,
path=None):
# type: (str, Optional[int], Optional[str]) -> None
"""For csvs, create a resource in the HDX datastore which enables data preview in HDX from a YAML file
containing a list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key.
If path is not supplied, the file is first downloaded from HDX.
Args:
yaml_path (str): Path to YAML file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'}
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(yaml_path)
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create_datastore_from_yaml_schema(self, yaml_path, delete_first=0,
path=None):
# type: (str, Optional[int], Optional[str]) -> None
"""For tabular data, create a resource in the HDX datastore which enables data preview in HDX from a YAML file
containing a list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key.
If path is not supplied, the file is first downloaded from HDX.
Args:
yaml_path (str): Path to YAML file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'}
delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0.
path (Optional[str]): Local path to file that was uploaded. Defaults to None.
Returns:
None
"""
data = load_yaml(yaml_path)
self.create_datastore_from_dict_schema(data, delete_first, path=path)
def __init__(self, **kwargs):
# type: (Any) -> None
email_config_found = False
email_config_dict = kwargs.get('email_config_dict', None)
if email_config_dict:
email_config_found = True
logger.info('Loading email configuration from dictionary')
email_config_json = kwargs.get('email_config_json', '')
if email_config_json:
if email_config_found:
raise EmailConfigurationError(
'More than one email configuration file given!')
email_config_found = True
logger.info('Loading email configuration from: %s' %
email_config_json)
email_config_dict = load_json(email_config_json)
email_config_yaml = kwargs.get('email_config_yaml', None)
if email_config_found:
if email_config_yaml:
raise EmailConfigurationError(
'More than one email configuration file given!')
else:
if not email_config_yaml:
logger.info(
'No email configuration parameter. Using default email configuration path.'
)
email_config_yaml = Email.default_email_config_yaml
logger.info('Loading email configuration from: %s' %
email_config_yaml)
email_config_dict = load_yaml(email_config_yaml)
self.connection_type = email_config_dict.get('connection_type', 'smtp')
self.host = email_config_dict.get('host', '')
self.port = email_config_dict.get('port', 0)
self.local_hostname = email_config_dict.get('local_hostname')
self.timeout = email_config_dict.get('timeout')
self.source_address = email_config_dict.get('source_address')
self.username = email_config_dict.get('username')
self.password = email_config_dict.get('password')
self.sender = email_config_dict.get('sender', self.username)
self.server = None
def __init__(self, **kwargs: Any) -> None:
email_config_found = False
email_config_dict = kwargs.get("email_config_dict", None)
if email_config_dict:
email_config_found = True
logger.info("Loading email configuration from dictionary")
email_config_json = kwargs.get("email_config_json", "")
if email_config_json:
if email_config_found:
raise EmailConfigurationError(
"More than one email configuration file given!")
email_config_found = True
logger.info(
f"Loading email configuration from: {email_config_json}")
email_config_dict = load_json(email_config_json)
email_config_yaml = kwargs.get("email_config_yaml", None)
if email_config_found:
if email_config_yaml:
raise EmailConfigurationError(
"More than one email configuration file given!")
else:
if not email_config_yaml:
logger.info(
"No email configuration parameter. Using default email configuration path."
)
email_config_yaml = Email.default_email_config_yaml
logger.info(
f"Loading email configuration from: {email_config_yaml}")
email_config_dict = load_yaml(email_config_yaml)
self.connection_type = email_config_dict.get("connection_type", "smtp")
self.host = email_config_dict.get("host", "")
self.port = email_config_dict.get("port", 0)
self.local_hostname = email_config_dict.get("local_hostname")
self.timeout = email_config_dict.get("timeout")
self.source_address = email_config_dict.get("source_address")
self.username = email_config_dict.get("username")
self.password = email_config_dict.get("password")
self.sender = email_config_dict.get("sender", self.username)
self.server = None
def _load(
cls,
prefix: str,
user_agent_config_yaml: str,
user_agent_lookup: Optional[str] = None,
) -> str:
"""
Load user agent YAML file
Args:
prefix (str): Text to put at start of user agent
user_agent_config_yaml (str): Path to user agent YAML file
user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied.
Returns:
str: user agent
"""
if not user_agent_config_yaml:
user_agent_config_yaml = cls.default_user_agent_config_yaml
logger.info(
f"No user agent or user agent config file given. Using default user agent config file: {user_agent_config_yaml}."
)
if not isfile(user_agent_config_yaml):
raise UserAgentError(
"User_agent should be supplied in a YAML config file. It can be your project's name for example."
)
logger.info(
f"Loading user agent config from: {user_agent_config_yaml}")
user_agent_config_dict = load_yaml(user_agent_config_yaml)
if user_agent_lookup:
user_agent_config_dict = user_agent_config_dict.get(
user_agent_lookup)
if not user_agent_config_dict:
raise UserAgentError(
f"No user agent information read from: {user_agent_config_yaml}"
)
ua = user_agent_config_dict.get("user_agent")
return cls._construct(user_agent_config_dict, prefix, ua)
'about': 'Data Scientist',
'apikey': '31e86726-2993-4d82-be93-3d2133c81d94',
'display_name': 'xxx',
'name': 'MyUser1',
'created': '2017-03-22T09:55:55.871573',
'email_hash': '9e2fc15d8cc65f43136f9dcef05fa184',
'email': '*****@*****.**',
'sysadmin': False,
'activity_streams_email_notifications': False,
'state': 'active',
'number_of_edits': 0,
'fullname': 'xxx xxx',
'id': '9f3e9973-7dbe-4c65-8820-f48578e3ffea',
'number_created_packages': 0
}
orgdict = load_yaml(join('tests', 'fixtures', 'organization_show_results.yml'))
orgdict2 = copy.deepcopy(orgdict)
del orgdict2['users']
del orgdict2['packages']
del orgdict2['extras']
orglist = [orgdict2]
result2dict = copy.deepcopy(resultdict)
result2dict['email'] = '*****@*****.**'
user_list = [resultdict, result2dict]
def user_mockshow(url, datadict):
if 'show' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_show"}')
def setup_logging(**kwargs) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get('smtp_config_dict', None)
if smtp_config_dict:
smtp_config_found = True
print('Loading smtp configuration customisations from dictionary')
smtp_config_json = kwargs.get('smtp_config_json', '')
if smtp_config_json:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_json)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get('smtp_config_yaml', '')
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_yaml)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get('logging_config_dict', None)
if logging_config_dict:
logging_config_found = True
print('Loading logging configuration from dictionary')
logging_config_json = kwargs.get('logging_config_json', '')
if logging_config_json:
if logging_config_found:
raise LoggingError(
'More than one logging configuration file given!')
logging_config_found = True
print('Loading logging configuration from: %s' % logging_config_json)
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get('logging_config_yaml', '')
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
'More than one logging configuration file given!')
else:
if not logging_config_yaml:
print('No logging configuration parameter. Using default.')
logging_config_yaml = script_dir_plus_file(
'logging_configuration.yml', setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
'logging_smtp_configuration.yml', setup_logging)
print('Loading base SMTP logging configuration from: %s' %
logging_smtp_config_yaml)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print('Loading logging configuration from: %s' % logging_config_yaml)
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict, logging_smtp_config_dict, smtp_config_dict
])
else:
raise LoggingError(
'SMTP logging configuration file given but not using default logging configuration!'
)
logging.config.dictConfig(logging_config_dict)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration file given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration file given!')
else:
if not hdx_config_yaml:
logger.info('No HDX configuration parameter. Using default.')
hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml',
Configuration)
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration file given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration file given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_config_dict,
project_config_dict)
self.hdx_read_only = kwargs.get('hdx_read_only', False)
if not self.hdx_read_only:
if 'hdx_key' in kwargs:
self.data['api_key'] = kwargs.get('hdx_key')
else:
hdx_key_file = kwargs.get('hdx_key_file',
join(expanduser('~'), '.hdxkey'))
""" :type : str"""
self.data['api_key'] = self.load_api_key(hdx_key_file)
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test')
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)
def setup_logging(**kwargs: Any) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get("smtp_config_dict", None)
if smtp_config_dict:
smtp_config_found = True
print("Loading smtp configuration customisations from dictionary")
smtp_config_json = kwargs.get("smtp_config_json", "")
if smtp_config_json:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_json}"
)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get("smtp_config_yaml", "")
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_yaml}"
)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get("logging_config_dict", None)
if logging_config_dict:
logging_config_found = True
print("Loading logging configuration from dictionary")
logging_config_json = kwargs.get("logging_config_json", "")
if logging_config_json:
if logging_config_found:
raise LoggingError(
"More than one logging configuration file given!")
logging_config_found = True
print(f"Loading logging configuration from: {logging_config_json}")
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get("logging_config_yaml", "")
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
"More than one logging configuration file given!")
else:
if not logging_config_yaml:
print("No logging configuration parameter. Using default.")
logging_config_yaml = script_dir_plus_file(
"logging_configuration.yml", setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
"logging_smtp_configuration.yml", setup_logging)
print(
f"Loading base SMTP logging configuration from: {logging_smtp_config_yaml}"
)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print(f"Loading logging configuration from: {logging_config_yaml}")
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict,
logging_smtp_config_dict,
smtp_config_dict,
])
else:
raise LoggingError(
"SMTP logging configuration file given but not using default logging configuration!"
)
file_only = os.getenv("LOG_FILE_ONLY")
if file_only is not None and file_only.lower() not in [
"false",
"f",
"n",
"no",
"0",
]:
root = logging_config_dict.get("root")
if root is not None:
handlers = root.get("handlers", list())
for i, handler in enumerate(handlers):
if handler.lower() == "console":
del handlers[i]
break
logging.config.dictConfig(logging_config_dict)
def get_session(
user_agent: Optional[str] = None,
user_agent_config_yaml: Optional[str] = None,
user_agent_lookup: Optional[str] = None,
use_env: bool = True,
fail_on_missing_file: bool = True,
**kwargs: Any,
) -> requests.Session:
"""Set up and return Session object that is set up with retrying. Requires either global user agent to be set or
appropriate user agent parameter(s) to be completed. If the EXTRA_PARAMS or BASIC_AUTH environment variable is
supplied, the extra_params* parameters will be ignored.
Args:
user_agent (Optional[str]): User agent string. HDXPythonUtilities/X.X.X- is prefixed.
user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied.
use_env (bool): Whether to read environment variables. Defaults to True.
fail_on_missing_file (bool): Raise an exception if any specified configuration files are missing. Defaults to True.
**kwargs: See below
auth (Tuple[str, str]): Authorisation information in tuple form (user, pass) OR
basic_auth (str): Authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) OR
basic_auth_file (str): Path to file containing authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx)
extra_params_dict (Dict): Extra parameters to put on end of url as a dictionary OR
extra_params_json (str): Path to JSON file containing extra parameters to put on end of url OR
extra_params_yaml (str): Path to YAML file containing extra parameters to put on end of url
extra_params_lookup (str): Lookup key for parameters. If not given assumes parameters are at root of the dict.
headers (Dict): Additional headers to add to request.
status_forcelist (iterable): HTTP statuses for which to force retry. Defaults to [429, 500, 502, 503, 504].
allowed_methods (iterable): HTTP methods for which to force retry. Defaults t0 frozenset(['GET']).
"""
s = requests.Session()
ua = kwargs.get("full_agent")
if not ua:
ua = UserAgent.get(user_agent, user_agent_config_yaml,
user_agent_lookup, **kwargs)
s.headers["User-Agent"] = ua
auths_found = list()
headers = kwargs.get("headers")
if headers is not None:
s.headers.update(headers)
if "Authorization" in headers:
auths_found.append("headers")
extra_params_found = False
extra_params_dict = None
basic_auth = None
if use_env:
basic_auth_env = os.getenv("BASIC_AUTH")
if basic_auth_env:
basic_auth = basic_auth_env
auths_found.append("basic_auth environment variable")
extra_params = os.getenv("EXTRA_PARAMS")
if extra_params:
if "=" in extra_params:
extra_params_dict = dict()
logger.info(
"Loading extra parameters from environment variable")
for extra_param in extra_params.split(","):
key, value = extra_param.split("=")
extra_params_dict[key] = value
extra_params_found = True
if not extra_params_found:
# only do this if extra params env vars not supplied
extra_params_dict = kwargs.get("extra_params_dict")
if extra_params_dict:
extra_params_found = True
logger.info("Loading extra parameters from dictionary")
extra_params_json = kwargs.get("extra_params_json", "")
if extra_params_json:
if extra_params_found:
raise SessionError(
"More than one set of extra parameters given!")
extra_params_found = True
logger.info(f"Loading extra parameters from: {extra_params_json}")
try:
extra_params_dict = load_json(extra_params_json)
except OSError:
if fail_on_missing_file:
raise
extra_params_yaml = kwargs.get("extra_params_yaml", "")
if extra_params_yaml:
if extra_params_found:
raise SessionError(
"More than one set of extra parameters given!")
logger.info(f"Loading extra parameters from: {extra_params_yaml}")
try:
extra_params_dict = load_yaml(extra_params_yaml)
except OSError:
if fail_on_missing_file:
raise
extra_params_lookup = kwargs.get("extra_params_lookup")
if extra_params_lookup and extra_params_dict:
extra_params_dict = extra_params_dict.get(extra_params_lookup)
if extra_params_dict is None:
raise SessionError(
f"{extra_params_lookup} does not exist in extra_params!")
if extra_params_dict:
basic_auth_param = extra_params_dict.get("basic_auth")
if basic_auth_param:
basic_auth = basic_auth_param
auths_found.append("basic_auth parameter")
del extra_params_dict["basic_auth"]
s.params = extra_params_dict
basic_auth_arg = kwargs.get("basic_auth")
if basic_auth_arg:
basic_auth = basic_auth_arg
auths_found.append("basic_auth argument")
auth = kwargs.get("auth")
if auth:
auths_found.append("auth argument")
basic_auth_file = kwargs.get("basic_auth_file")
if basic_auth_file:
logger.info(f"Loading basic auth from: {basic_auth_file}")
try:
basic_auth = load_file_to_str(basic_auth_file, strip=True)
auths_found.append(f"file {basic_auth_file}")
except OSError:
if fail_on_missing_file:
raise
if len(auths_found) > 1:
auths_found_str = ", ".join(auths_found)
raise SessionError(
f"More than one authorisation given! ({auths_found_str})")
if "headers" not in auths_found:
if basic_auth:
auth = decode(basic_auth)
s.auth = auth
status_forcelist = kwargs.get("status_forcelist",
[429, 500, 502, 503, 504])
allowed_methods = kwargs.get(
"allowed_methods",
frozenset(["HEAD", "TRACE", "GET", "PUT", "OPTIONS", "DELETE"]),
)
retries = Retry(
total=5,
backoff_factor=0.4,
status_forcelist=status_forcelist,
allowed_methods=allowed_methods,
raise_on_redirect=True,
raise_on_status=True,
)
s.mount("file://", FileAdapter())
s.mount(
"http://",
HTTPAdapter(max_retries=retries,
pool_connections=100,
pool_maxsize=100),
)
s.mount(
"https://",
HTTPAdapter(max_retries=retries,
pool_connections=100,
pool_maxsize=100),
)
return s
'visual.png',
'revision_id':
'cc64364b-ede2-400a-bb9f-8e585a4f6399',
'notes':
'My Showcase',
'url':
'http://visualisation/url/',
'title':
'MyShowcase1',
'image_display_url':
'http://myvisual/visual.png',
'name':
'showcase-1'
}
datasetsdict = load_yaml(join('tests', 'fixtures', 'dataset_all_results.yml'))
def mockshow(url, datadict):
if 'package_list' in url:
result = json.dumps(datasetsdict)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_list"}'
% result)
if '_show' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_show"}'
)
result = json.dumps(showcase_resultdict)
import copy
import json
from os.path import join
import pytest
from hdx.utilities.dictandlist import merge_two_dictionaries
from hdx.utilities.loader import load_yaml
from hdx.data.hdxobject import HDXError
from hdx.data.organization import Organization
from hdx.data.user import User
from hdx.hdx_configuration import Configuration
from . import MockResponse, organization_data, user_data
from .test_user import user_mockshow
resultdict = load_yaml(join('tests', 'fixtures', 'organization_show_results.yml'))
organization_list = ['acaps', 'accion-contra-el-hambre-acf-spain', 'acf-west-africa', 'acled', 'acted',
'action-contre-la-faim', 'adeso', 'afd', 'afdb', 'afghanistan-protection-cluster']
searchdict = load_yaml(join('tests', 'fixtures', 'dataset_search_results.yml'))
def organization_mockshow(url, datadict):
if 'show' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_show"}')
result = json.dumps(resultdict)
if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19' or datadict['id'] == 'TEST1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_show"}' % result)
if datadict['id'] == 'TEST2':
def config(self):
return load_yaml(join('tests', 'fixtures', 'adminone.yml'))
'2016-06-09T12:49:33.854367',
'groups':
resultgroups,
'data_update_frequency':
'7',
'tags':
resulttags,
'version':
None,
'solr_additions':
'{"countries": ["Algeria", "Zimbabwe"]}',
'dataset_date':
'06/04/2016'
}
searchdict = load_yaml(join('fixtures', 'search_results.yml'))
def mockshow(url, datadict):
if 'show' not in url and 'related_list' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}'
)
result = json.dumps(resultdict)
if 'related_list' in url:
result = json.dumps(TestDataset.gallery_data)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=related_list"}'
% result)
def main(file_path, hdx_key, user_agent, preprefix, hdx_site, db_url, db_params, gsheet_auth):
if db_params:
params = args_to_dict(db_params)
elif db_url:
params = Database.get_params_from_sqlalchemy_url(db_url)
else:
params = {'driver': 'sqlite', 'database': 'freshness.db'}
logger.info('> Database parameters: %s' % params)
with Database(**params) as session:
info = json.loads(gsheet_auth)
scopes = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive']
credentials = service_account.Credentials.from_service_account_info(info, scopes=scopes)
gc = pygsheets.authorize(custom_credentials=credentials)
configuration = load_yaml('project_configuration.yml')
spreadsheet = gc.open_by_url(configuration['spreadsheet_url'])
sheet = spreadsheet.worksheet_by_title('datasets')
sheet.clear()
rows = [['update freq', 'fresh', 'no days', 'title', 'run date', 'last modified', 'dataset date', 'dataset end date', 'org title', 'URL', 'id', 'org id', 'maintainer', 'what updated', 'resources']]
run_number, run_date = session.query(DBRun.run_number, DBRun.run_date).order_by(DBRun.run_number.desc()).first()
logger.info('Run number is %d' % run_number)
datasetcolumns = [DBDataset.update_frequency, DBDataset.fresh, DBInfoDataset.title, DBDataset.last_modified,
DBDataset.dataset_date, DBOrganization.title.label('organization_title'), DBInfoDataset.name,
DBDataset.id, DBOrganization.id.label('organization_id'), DBInfoDataset.maintainer, DBDataset.what_updated]
resourcecolumns = [DBDataset.id, DBResource.url]
def get_datasets(update_frequency, fresh):
filters = [DBDataset.run_number == run_number, DBDataset.id == DBInfoDataset.id,
DBInfoDataset.organization_id == DBOrganization.id,
DBDataset.fresh == fresh, DBDataset.update_frequency == update_frequency]
return session.query(*datasetcolumns).filter(and_(*filters))
def get_resources(dataset_ids):
filters = [DBDataset.run_number == run_number, DBResource.run_number == run_number,
DBDataset.id == DBResource.dataset_id, DBDataset.id.in_(dataset_ids)]
return session.query(*resourcecolumns).filter(and_(*filters))
fresh_values = [0, 1, 2, 3]
update_frequencies = [1, 7, 14, 30, 180, 365]
repobase = '%s/tree/master/datasets/' % configuration['repo']
dir = join(file_path, 'datasets')
rmtree(dir, ignore_errors=True)
mkdir(dir)
with Download(user_agent=user_agent, preprefix=preprefix) as downloader:
status_forcelist = [429, 500, 502, 503, 504]
method_whitelist = frozenset(['HEAD', 'TRACE', 'GET', 'PUT', 'OPTIONS', 'DELETE'])
retries = Retry(total=1, backoff_factor=0.4, status_forcelist=status_forcelist,
method_whitelist=method_whitelist,
raise_on_redirect=True,
raise_on_status=True)
downloader.session.mount('http://', HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100))
downloader.session.mount('https://', HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100))
for update_frequency in update_frequencies:
for fresh in fresh_values:
org_ids = list()
results = get_datasets(update_frequency, fresh)
datasets = list()
ids = list()
datasets_urls = dict()
for dataset in results:
dataset = list(dataset)
datasets.append(dataset)
ids.append(dataset[7])
for result in get_resources(ids):
resource = list(result)
dict_of_lists_add(datasets_urls, resource[0], resource[1])
for dataset in datasets:
org_id = dataset[8]
if org_id in org_ids:
continue
dataset = list(dataset)
dataset[0] = Dataset.transform_update_frequency(str(update_frequency))
fresh = dataset[1]
if fresh == 0:
dataset[1] = 'fresh'
elif fresh == 1:
dataset[1] = 'due'
elif fresh == 2:
dataset[1] = 'overdue'
elif fresh == 3:
dataset[1] = 'delinquent'
last_modified = dataset[3]
dataset[3] = last_modified.isoformat()
nodays = (run_date - last_modified).days
dataset.insert(2, nodays)
dataset.insert(4, run_date.isoformat())
dataset_date = dataset[6]
if '-' in dataset_date:
dataset_date = dataset_date.split('-')
dataset[6] = datetime.strptime(dataset_date[0], '%m/%d/%Y').date().isoformat()
dataset.insert(7, datetime.strptime(dataset_date[1], '%m/%d/%Y').date().isoformat())
else:
dataset[6] = datetime.strptime(dataset_date, '%m/%d/%Y').date().isoformat()
dataset.insert(7, '')
dataset_name = dataset[9]
dataset[9] = 'https://data.humdata.org/dataset/%s' % dataset_name
org_ids.append(org_id)
if len(org_ids) == 6:
break
urls = datasets_urls[dataset[10]]
if len(urls) != 0:
datasetdir = join(dir, dataset_name)
mkdir(datasetdir)
for url in urls:
urlpath = urlsplit(url).path
filename = basename(urlpath)
try:
downloader.download_file(url, datasetdir, filename)
except DownloadError as ex:
with open(join(datasetdir, filename), 'w') as text_file:
text_file.write(str(ex))
dataset.append('%s%s' % (repobase, dataset_name))
else:
dataset.append('')
rows.append(dataset)
logger.info('Added dataset %s' % dataset_name)
sheet.update_values('A1', rows)
],
'groups': [],
'relationships_as_subject': [],
'total_res_downloads': 0,
'num_datasets': 2,
'showcase_notes_formatted': '<p>lalala</p>',
'image_url': 'visual.png',
'revision_id': 'cc64364b-ede2-400a-bb9f-8e585a4f6399',
'notes': 'My Showcase',
'url': 'http://visualisation/url/',
'title': 'MyShowcase1',
'image_display_url': 'http://myvisual/visual.png',
'name': 'showcase-1'
}
datasetsdict = load_yaml(join('tests', 'fixtures', 'dataset_all_results.yml'))
def mockshow(url, datadict):
if 'package_list' in url:
result = json.dumps(datasetsdict)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_list"}' % result)
if '_show' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_show"}')
result = json.dumps(showcase_resultdict)
if datadict['id'] == '05e392bf-04e0-4ca6-848c-4e87bba10746' or datadict['id'] == 'TEST1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_show"}' % result)
if datadict['id'] == 'TEST2':
def test_load_yaml_into_existing_dict(self, configfolder):
existing_dict = load_yaml(join(configfolder, "hdx_config.yml"))
result = load_yaml_into_existing_dict(
existing_dict, join(configfolder, "project_configuration.yml")
)
assert list(result.items()) == list(TestLoader.expected_yaml.items())
'about': 'Data Scientist',
'apikey': '31e86726-2993-4d82-be93-3d2133c81d94',
'display_name': 'xxx',
'name': 'MyUser1',
'created': '2017-03-22T09:55:55.871573',
'email_hash': '9e2fc15d8cc65f43136f9dcef05fa184',
'email': '*****@*****.**',
'sysadmin': False,
'activity_streams_email_notifications': False,
'state': 'active',
'number_of_edits': 0,
'fullname': 'xxx xxx',
'id': '9f3e9973-7dbe-4c65-8820-f48578e3ffea',
'number_created_packages': 0
}
orgdict = load_yaml(join('tests', 'fixtures', 'organization_show_results.yml'))
orgdict2 = copy.deepcopy(orgdict)
del orgdict2['users']
del orgdict2['packages']
del orgdict2['extras']
orglist = [orgdict2]
result2dict = copy.deepcopy(resultdict)
result2dict['email'] = '*****@*****.**'
user_list = [resultdict, result2dict]
def user_mockshow(url, datadict):
if 'show' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_show"}')
def test_load_empty(self, fixturesfolder):
loaderfolder = join(fixturesfolder, 'loader')
with pytest.raises(LoadError):
load_yaml(join(loaderfolder, 'empty.yml'))
with pytest.raises(LoadError):
load_json(join(loaderfolder, 'empty.json'))
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
self._remoteckan = None
self._emailer = None
hdx_base_config_found = False
hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None)
if hdx_base_config_dict:
hdx_base_config_found = True
logger.info('Loading HDX base configuration from dictionary')
hdx_base_config_json = kwargs.get('hdx_base_config_json', '')
if hdx_base_config_json:
if hdx_base_config_found:
raise ConfigurationError('More than one HDX base configuration given!')
hdx_base_config_found = True
logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json)
hdx_base_config_dict = load_json(hdx_base_config_json)
hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '')
if hdx_base_config_found:
if hdx_base_config_yaml:
raise ConfigurationError('More than one HDX base configuration given!')
else:
if not hdx_base_config_yaml:
hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml
logger.info('No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml)
logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml)
hdx_base_config_dict = load_yaml(hdx_base_config_yaml)
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError('More than one HDX configuration given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError('More than one HDX configuration given!')
else:
if not hdx_config_yaml:
hdx_config_yaml = Configuration.default_hdx_config_yaml
if isfile(hdx_config_yaml):
logger.info('No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml)
else:
logger.info('No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml)
hdx_config_yaml = None
hdx_config_dict = dict()
if hdx_config_yaml:
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError('More than one project configuration given!')
project_config_found = True
logger.info('Loading project configuration from: %s' % project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError('More than one project configuration given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' % project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict)
ua = kwargs.get('full_agent')
if ua:
self.user_agent = ua
else:
try:
self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs)
except UserAgentError:
self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data)
self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False))
logger.info('Read only access to HDX: %s' % str(self.hdx_read_only))
self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key'))
if not self.hdx_key and not self.hdx_read_only:
raise ConfigurationError('No HDX API key supplied as a parameter or in configuration!')
hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url'))
if hdx_url:
self.hdx_site = 'hdx_custom_site'
self.data[self.hdx_site] = {'url': hdx_url}
else:
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', self.data.get('hdx_site', 'test'))
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def __init__(self, **kwargs):
super(Configuration, self).__init__()
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError('More than one HDX configuration file given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError('More than one HDX configuration file given!')
else:
if not hdx_config_yaml:
logger.info('No HDX configuration parameter. Using default.')
hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration)
logger.info('Loading HDX configuration from: %s' % hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError('More than one project configuration file given!')
project_config_found = True
logger.info('Loading project configuration from: %s' % project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError('More than one project configuration file given!')
else:
if not project_config_yaml:
logger.info('No project configuration parameter. Using default.')
project_config_yaml = join('config', 'project_configuration.yml')
logger.info('Loading project configuration from: %s' % project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict)
hdx_key_file = kwargs.get('hdx_key_file', join('%s' % expanduser("~"), '.hdxkey'))
self.data['api_key'] = self.load_api_key(hdx_key_file)
self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test')
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def __init__(self, **kwargs):
# type: (Any) -> None
super(Configuration, self).__init__()
self._remoteckan = None
self._emailer = None
hdx_base_config_found = False
hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None)
if hdx_base_config_dict:
hdx_base_config_found = True
logger.info('Loading HDX base configuration from dictionary')
hdx_base_config_json = kwargs.get('hdx_base_config_json', '')
if hdx_base_config_json:
if hdx_base_config_found:
raise ConfigurationError(
'More than one HDX base configuration given!')
hdx_base_config_found = True
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_json)
hdx_base_config_dict = load_json(hdx_base_config_json)
hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '')
if hdx_base_config_found:
if hdx_base_config_yaml:
raise ConfigurationError(
'More than one HDX base configuration given!')
else:
if not hdx_base_config_yaml:
hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml
logger.info(
'No HDX base configuration parameter. Using default base configuration file: %s.'
% hdx_base_config_yaml)
logger.info('Loading HDX base configuration from: %s' %
hdx_base_config_yaml)
hdx_base_config_dict = load_yaml(hdx_base_config_yaml)
hdx_config_found = False
hdx_config_dict = kwargs.get('hdx_config_dict', None)
if hdx_config_dict:
hdx_config_found = True
logger.info('Loading HDX configuration from dictionary')
hdx_config_json = kwargs.get('hdx_config_json', '')
if hdx_config_json:
if hdx_config_found:
raise ConfigurationError(
'More than one HDX configuration given!')
hdx_config_found = True
logger.info('Loading HDX configuration from: %s' % hdx_config_json)
hdx_config_dict = load_json(hdx_config_json)
hdx_config_yaml = kwargs.get('hdx_config_yaml', '')
if hdx_config_found:
if hdx_config_yaml:
raise ConfigurationError(
'More than one HDX configuration given!')
else:
if not hdx_config_yaml:
hdx_config_yaml = Configuration.default_hdx_config_yaml
if isfile(hdx_config_yaml):
logger.info(
'No HDX configuration parameter. Using default configuration file: %s.'
% hdx_config_yaml)
else:
logger.info(
'No HDX configuration parameter and no configuration file at default path: %s.'
% hdx_config_yaml)
hdx_config_yaml = None
hdx_config_dict = dict()
if hdx_config_yaml:
logger.info('Loading HDX configuration from: %s' %
hdx_config_yaml)
hdx_config_dict = load_yaml(hdx_config_yaml)
self.data = merge_two_dictionaries(hdx_base_config_dict,
hdx_config_dict)
project_config_found = False
project_config_dict = kwargs.get('project_config_dict', None)
if project_config_dict is not None:
project_config_found = True
logger.info('Loading project configuration from dictionary')
project_config_json = kwargs.get('project_config_json', '')
if project_config_json:
if project_config_found:
raise ConfigurationError(
'More than one project configuration given!')
project_config_found = True
logger.info('Loading project configuration from: %s' %
project_config_json)
project_config_dict = load_json(project_config_json)
project_config_yaml = kwargs.get('project_config_yaml', '')
if project_config_found:
if project_config_yaml:
raise ConfigurationError(
'More than one project configuration given!')
else:
if project_config_yaml:
logger.info('Loading project configuration from: %s' %
project_config_yaml)
project_config_dict = load_yaml(project_config_yaml)
else:
project_config_dict = dict()
self.data = merge_two_dictionaries(hdx_base_config_dict,
project_config_dict)
ua = kwargs.get('full_agent')
if ua:
self.user_agent = ua
else:
try:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**kwargs)
except UserAgentError:
self.user_agent = UserAgent.get(prefix=Configuration.prefix,
**self.data)
self.hdx_read_only = kwargs.get('hdx_read_only',
self.data.get('hdx_read_only', False))
logger.info('Read only access to HDX: %s' % str(self.hdx_read_only))
self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key'))
if not self.hdx_key and not self.hdx_read_only:
raise ConfigurationError(
'No HDX API key supplied as a parameter or in configuration!')
hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url'))
if hdx_url:
self.hdx_site = 'hdx_custom_site'
hdx_url = hdx_url.rstrip('/')
self.data[self.hdx_site] = {'url': hdx_url}
else:
self.hdx_site = 'hdx_%s_site' % kwargs.get(
'hdx_site', self.data.get('hdx_site', 'test'))
if self.hdx_site not in self.data:
raise ConfigurationError('%s not defined in configuration!' %
self.hdx_site)
def test_load_yaml_into_existing_dict(self, configfolder):
existing_dict = load_yaml(join(configfolder, 'hdx_config.yml'))
result = load_yaml_into_existing_dict(
existing_dict, join(configfolder, 'project_configuration.yml'))
assert result == TestLoader.expected_yaml
import copy
import json
from os.path import join
import pytest
from hdx.utilities.dictandlist import merge_two_dictionaries
from hdx.utilities.loader import load_yaml
from hdx.data.hdxobject import HDXError
from hdx.data.organization import Organization
from hdx.data.user import User
from hdx.hdx_configuration import Configuration
from . import MockResponse, organization_data, user_data
from .test_user import user_mockshow
resultdict = load_yaml(
join('tests', 'fixtures', 'organization_show_results.yml'))
organization_list = [
'acaps', 'accion-contra-el-hambre-acf-spain', 'acf-west-africa', 'acled',
'acted', 'action-contre-la-faim', 'adeso', 'afd', 'afdb',
'afghanistan-protection-cluster'
]
searchdict = load_yaml(join('tests', 'fixtures', 'dataset_search_results.yml'))
def organization_mockshow(url, datadict):
if 'show' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_show"}'
)
'visual.png',
'revision_id':
'cc64364b-ede2-400a-bb9f-8e585a4f6399',
'notes':
'My Showcase',
'url':
'http://visualisation/url/',
'title':
'MyShowcase1',
'image_display_url':
'http://myvisual/visual.png',
'name':
'showcase-1'
}
datasetsdict = load_yaml(
join('tests', 'fixtures', 'dataset_search_results.yml'))
allsearchdict = load_yaml(
join('tests', 'fixtures', 'showcase_all_search_results.yml'))
def mockshow(url, datadict):
if 'package_list' in url:
result = json.dumps(datasetsdict['results'])
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_list"}'
% result)
if '_show' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_show"}'
def setup_logging(**kwargs) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get('smtp_config_dict', None)
if smtp_config_dict:
smtp_config_found = True
print('Loading smtp configuration customisations from dictionary')
smtp_config_json = kwargs.get('smtp_config_json', '')
if smtp_config_json:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' % smtp_config_json)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get('smtp_config_yaml', '')
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' % smtp_config_yaml)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get('logging_config_dict', None)
if logging_config_dict:
logging_config_found = True
print('Loading logging configuration from dictionary')
logging_config_json = kwargs.get('logging_config_json', '')
if logging_config_json:
if logging_config_found:
raise LoggingError('More than one logging configuration file given!')
logging_config_found = True
print('Loading logging configuration from: %s' % logging_config_json)
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get('logging_config_yaml', '')
if logging_config_found:
if logging_config_yaml:
raise LoggingError('More than one logging configuration file given!')
else:
if not logging_config_yaml:
print('No logging configuration parameter. Using default.')
logging_config_yaml = script_dir_plus_file('logging_configuration.yml', setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file('logging_smtp_configuration.yml', setup_logging)
print('Loading base SMTP logging configuration from: %s' % logging_smtp_config_yaml)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print('Loading logging configuration from: %s' % logging_config_yaml)
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([logging_config_dict, logging_smtp_config_dict, smtp_config_dict])
else:
raise LoggingError('SMTP logging configuration file given but not using default logging configuration!')
logging.config.dictConfig(logging_config_dict)
def get_session(**kwargs):
# type: (Any) -> requests.Session
"""Set up and return Session object that is set up with retrying
Args:
**kwargs: See below
auth (Tuple[str, str]): Authorisation information in tuple form (user, pass) OR
basic_auth (str): Authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) OR
basic_auth_file (str): Path to file containing authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx)
extra_params_dict (Dict): Extra parameters to put on end of url as a dictionary OR
extra_params_json (str): Path to JSON file containing extra parameters to put on end of url OR
extra_params_yaml (str): Path to YAML file containing extra parameters to put on end of url
extra_params_lookup (str): Lookup key for parameters. If not given assumes parameters are at root of the dict.
status_forcelist (iterable): HTTP statuses for which to force retry. Defaults to [429, 500, 502, 503, 504].
method_whitelist (iterable): HTTP methods for which to force retry. Defaults t0 frozenset(['GET']).
"""
s = requests.Session()
extra_params_found = False
extra_params_dict = kwargs.get('extra_params_dict', None)
if extra_params_dict:
extra_params_found = True
logger.info('Loading extra parameters from dictionary')
extra_params_json = kwargs.get('extra_params_json', '')
if extra_params_json:
if extra_params_found:
raise SessionError('More than one set of extra parameters given!')
extra_params_found = True
logger.info('Loading extra parameters from: %s' % extra_params_json)
extra_params_dict = load_json(extra_params_json)
extra_params_yaml = kwargs.get('extra_params_yaml', '')
if extra_params_found:
if extra_params_yaml:
raise SessionError('More than one set of extra parameters given!')
else:
if extra_params_yaml:
logger.info('Loading extra parameters from: %s' %
extra_params_yaml)
extra_params_dict = load_yaml(extra_params_yaml)
else:
extra_params_dict = dict()
extra_params_lookup = kwargs.get('extra_params_lookup')
if extra_params_lookup:
extra_params_dict = extra_params_dict.get(extra_params_lookup)
if extra_params_dict is None:
raise SessionError('%s does not exist in extra_params!' %
extra_params_lookup)
auth_found = False
basic_auth = extra_params_dict.get('basic_auth')
if basic_auth:
logger.info('Loading authorisation from basic_auth parameter')
auth_found = True
del extra_params_dict['basic_auth']
s.params = extra_params_dict
auth = kwargs.get('auth')
if auth:
if auth_found:
raise SessionError('More than one authorisation given!')
logger.info('Loading authorisation from auth argument')
auth_found = True
bauth = kwargs.get('basic_auth')
if bauth:
if auth_found:
raise SessionError('More than one authorisation given!')
logger.info('Loading authorisation from basic_auth argument')
basic_auth = bauth
auth_found = True
basic_auth_file = kwargs.get('basic_auth_file')
if basic_auth_file:
if auth_found:
raise SessionError('More than one authorisation given!')
logger.info('Loading authorisation from: %s' % basic_auth_file)
basic_auth = load_file_to_str(basic_auth_file)
if basic_auth:
auth = decode(basic_auth)
s.auth = auth
status_forcelist = kwargs.get('status_forcelist',
[429, 500, 502, 503, 504])
method_whitelist = kwargs.get(
'method_whitelist',
frozenset(['HEAD', 'TRACE', 'GET', 'PUT', 'OPTIONS', 'DELETE']))
retries = Retry(total=5,
backoff_factor=0.4,
status_forcelist=status_forcelist,
method_whitelist=method_whitelist,
raise_on_redirect=True,
raise_on_status=True)
s.mount(
'http://',
HTTPAdapter(max_retries=retries,
pool_connections=100,
pool_maxsize=100))
s.mount(
'https://',
HTTPAdapter(max_retries=retries,
pool_connections=100,
pool_maxsize=100))
return s
