def test_load_empty(self, fixturesfolder): loaderfolder = join(fixturesfolder, "loader") with pytest.raises(LoadError): load_file_to_str(join(loaderfolder, "empty.yml")) with pytest.raises(LoadError): load_yaml(join(loaderfolder, "empty.yml")) with pytest.raises(LoadError): load_json(join(loaderfolder, "empty.json"))
def retrieve_json( self, url, filename, logstr=None, fallback=False, **kwargs ): """Retrieve JSON Args: url (str): URL to download filename (str): Filename to use for saved file logstr (Optional[str]): Text to use in log string to describe download. Defaults to filename. fallback (bool): Whether to use static fallback if download fails. Defaults to False. **kwargs: Parameters to pass to download call Returns: Union[Dict,List]: The data from the JSON file """ if not logstr: logstr = filename saved_path = join(self.saved_dir, filename) if self.use_saved: logger.info(f"Using saved {logstr} in {saved_path}") rjson = load_json(saved_path) else: try: logger.info( f"Downloading {logstr} from {self.get_url_logstr(url)}" ) self.downloader.download(url, **kwargs) rjson = self.downloader.get_json() if self.save: logger.info(f"Saving {logstr} in {saved_path}") save_json(rjson, saved_path) except DownloadError: if not fallback: raise fallback_path = join(self.fallback_dir, filename) logger.exception( f"{logstr} download failed, using static data {fallback_path}!" ) rjson = load_json(fallback_path) return rjson
def setup(cls, configuration): logger.info("Reading in lookups data") org_data = load_json(configuration["org_data"]) """ Map from IATI identifiers to organisation names """ # Prime with org identifiers from code4iati for entry in org_data["data"]: code = clean_string(entry["code"]).lower() name = clean_string(entry["name"]) cls.org_ref_to_name[code] = name cls.org_names_to_ref[name.lower()] = code cls.sector_info = load_json(configuration["sector_data"]) region_data = load_json(configuration["region_data"]) """ Map from region codes to region names """ # Prime with region codes from code4iati for entry in region_data["data"]: code = clean_string(entry["code"]).lower() name = clean_region(entry["name"]) cls.region_code_to_name[code] = name cls.default_org_id = configuration["default_org_id"] cls.default_org_name = configuration["default_org_name"] cls.default_expenditure_org_name = configuration[ "default_expenditure_org_name"] cls.default_sector = configuration["default_sector"] cls.default_country_region = configuration["default_country_region"] for row in hxl.data(configuration["filters_url"]): org_id = row.get("#org+reporting+id") if org_id: cls.filter_reporting_orgs.append(org_id) org_id = row.get("#org+reporting_children+id") hierarchy = row.get("#org+reporting_children+hierarchy") if org_id and hierarchy: cls.filter_reporting_orgs_children[org_id] = hierarchy for row in hxl.data(configuration["blocklist_url"]): org_id = row.get("#org+reporting+id") if org_id: cls.org_ref_blocklist.append(org_id)
def create_datastore_from_json_schema(self, path: str, delete_first: int = 0) -> None: """Creates a resource in the HDX datastore from a JSON file containing a list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key Args: path (str): Path to JSON file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. Returns: None """ data = load_json(path) self.create_datastore_from_dict_schema(data, delete_first)
def create_datastore_from_json_schema(self, json_path, delete_first=0, path=None): # type: (str, int, Optional[str]) -> None """For csvs, create a resource in the HDX datastore which enables data preview in HDX from a JSON file containing a list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key. If path is not supplied, the file is first downloaded from HDX. Args: json_path (str): Path to JSON file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. path (Optional[str]): Local path to file that was uploaded. Defaults to None. Returns: None """ data = load_json(json_path) self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create_datastore_from_json_schema(self, json_path, delete_first=0, path=None): # type: (str, int, Optional[str]) -> None """For tabular data, create a resource in the HDX datastore which enables data preview in HDX from a JSON file containing a list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} and optionally a primary key. If path is not supplied, the file is first downloaded from HDX. Args: json_path (str): Path to JSON file containing list of fields and types of form {'id': 'FIELD', 'type': 'TYPE'} delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. path (Optional[str]): Local path to file that was uploaded. Defaults to None. Returns: None """ data = load_json(json_path) self.create_datastore_from_dict_schema(data, delete_first, path=path)
def __init__(self, **kwargs): # type: (Any) -> None email_config_found = False email_config_dict = kwargs.get('email_config_dict', None) if email_config_dict: email_config_found = True logger.info('Loading email configuration from dictionary') email_config_json = kwargs.get('email_config_json', '') if email_config_json: if email_config_found: raise EmailConfigurationError( 'More than one email configuration file given!') email_config_found = True logger.info('Loading email configuration from: %s' % email_config_json) email_config_dict = load_json(email_config_json) email_config_yaml = kwargs.get('email_config_yaml', None) if email_config_found: if email_config_yaml: raise EmailConfigurationError( 'More than one email configuration file given!') else: if not email_config_yaml: logger.info( 'No email configuration parameter. Using default email configuration path.' ) email_config_yaml = Email.default_email_config_yaml logger.info('Loading email configuration from: %s' % email_config_yaml) email_config_dict = load_yaml(email_config_yaml) self.connection_type = email_config_dict.get('connection_type', 'smtp') self.host = email_config_dict.get('host', '') self.port = email_config_dict.get('port', 0) self.local_hostname = email_config_dict.get('local_hostname') self.timeout = email_config_dict.get('timeout') self.source_address = email_config_dict.get('source_address') self.username = email_config_dict.get('username') self.password = email_config_dict.get('password') self.sender = email_config_dict.get('sender', self.username) self.server = None
def __init__(self, **kwargs: Any) -> None: email_config_found = False email_config_dict = kwargs.get("email_config_dict", None) if email_config_dict: email_config_found = True logger.info("Loading email configuration from dictionary") email_config_json = kwargs.get("email_config_json", "") if email_config_json: if email_config_found: raise EmailConfigurationError( "More than one email configuration file given!") email_config_found = True logger.info( f"Loading email configuration from: {email_config_json}") email_config_dict = load_json(email_config_json) email_config_yaml = kwargs.get("email_config_yaml", None) if email_config_found: if email_config_yaml: raise EmailConfigurationError( "More than one email configuration file given!") else: if not email_config_yaml: logger.info( "No email configuration parameter. Using default email configuration path." ) email_config_yaml = Email.default_email_config_yaml logger.info( f"Loading email configuration from: {email_config_yaml}") email_config_dict = load_yaml(email_config_yaml) self.connection_type = email_config_dict.get("connection_type", "smtp") self.host = email_config_dict.get("host", "") self.port = email_config_dict.get("port", 0) self.local_hostname = email_config_dict.get("local_hostname") self.timeout = email_config_dict.get("timeout") self.source_address = email_config_dict.get("source_address") self.username = email_config_dict.get("username") self.password = email_config_dict.get("password") self.sender = email_config_dict.get("sender", self.username) self.server = None
def schemacreator(start_url, base_url, template_path, output_folder): with Download() as downloader: response = downloader.download(start_url) countryisos = set() for service in response.json()['services']: servicenameright = service['name'].split('/')[1] iso3 = servicenameright[:3] countryisos.add(iso3) template = load_json(template_path) for iso3 in sorted(countryisos): print(iso3) url = base_url % iso3 response = downloader.download(url) adminlevels = set() for layer in response.json()['layers']: layername = layer['name'].lower() if 'feature' in layer['type'].lower() and 'admin' in layername and not 'lines' in layername: adminlevel = int(layername[-1]) if adminlevel > 0: adminlevels.add(adminlevel) adminlevels = sorted(list(adminlevels)) print(adminlevels) schema = list() admrules = list() for rule_template in template: if '{ADM}' in str(rule_template): admrules.append(rule_template) else: for adminlevel in adminlevels: for admrule in admrules: schema.append(get_rule(admrule, iso3, adminlevel)) admrules = list() schema.append(get_rule(rule_template, iso3)) for adminlevel in adminlevels: for admrule in admrules: schema.append(get_rule(admrule, iso3, adminlevel)) save_json(schema, join(output_folder, 'validation-schema-pcodes-%s.json' % iso3.lower()), pretty=True)
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration file given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration file given!') else: if not hdx_config_yaml: logger.info('No HDX configuration parameter. Using default.') hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration) logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration file given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration file given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict) self.hdx_read_only = kwargs.get('hdx_read_only', False) if not self.hdx_read_only: if 'hdx_key' in kwargs: self.data['api_key'] = kwargs.get('hdx_key') else: hdx_key_file = kwargs.get('hdx_key_file', join(expanduser('~'), '.hdxkey')) """ :type : str""" self.data['api_key'] = self.load_api_key(hdx_key_file) self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test') if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def setup_logging(**kwargs) -> None: """Setup logging configuration Args: **kwargs: See below logging_config_dict (dict): Logging configuration dictionary OR logging_config_json (str): Path to JSON Logging configuration OR logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml. smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration Returns: None """ smtp_config_found = False smtp_config_dict = kwargs.get('smtp_config_dict', None) if smtp_config_dict: smtp_config_found = True print('Loading smtp configuration customisations from dictionary') smtp_config_json = kwargs.get('smtp_config_json', '') if smtp_config_json: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_json) smtp_config_dict = load_json(smtp_config_json) smtp_config_yaml = kwargs.get('smtp_config_yaml', '') if smtp_config_yaml: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_yaml) smtp_config_dict = load_yaml(smtp_config_yaml) logging_smtp_config_dict = None logging_config_found = False logging_config_dict = kwargs.get('logging_config_dict', None) if logging_config_dict: logging_config_found = True print('Loading logging configuration from dictionary') logging_config_json = kwargs.get('logging_config_json', '') if logging_config_json: if logging_config_found: raise LoggingError( 'More than one logging configuration file given!') logging_config_found = True print('Loading logging configuration from: %s' % logging_config_json) logging_config_dict = load_json(logging_config_json) logging_config_yaml = kwargs.get('logging_config_yaml', '') if logging_config_found: if logging_config_yaml: raise LoggingError( 'More than one logging configuration file given!') else: if not logging_config_yaml: print('No logging configuration parameter. Using default.') logging_config_yaml = script_dir_plus_file( 'logging_configuration.yml', setup_logging) if smtp_config_found: logging_smtp_config_yaml = script_dir_plus_file( 'logging_smtp_configuration.yml', setup_logging) print('Loading base SMTP logging configuration from: %s' % logging_smtp_config_yaml) logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml) print('Loading logging configuration from: %s' % logging_config_yaml) logging_config_dict = load_yaml(logging_config_yaml) if smtp_config_found: if logging_smtp_config_dict: logging_config_dict = merge_dictionaries([ logging_config_dict, logging_smtp_config_dict, smtp_config_dict ]) else: raise LoggingError( 'SMTP logging configuration file given but not using default logging configuration!' ) logging.config.dictConfig(logging_config_dict)
def get_session( user_agent: Optional[str] = None, user_agent_config_yaml: Optional[str] = None, user_agent_lookup: Optional[str] = None, use_env: bool = True, fail_on_missing_file: bool = True, **kwargs: Any, ) -> requests.Session: """Set up and return Session object that is set up with retrying. Requires either global user agent to be set or appropriate user agent parameter(s) to be completed. If the EXTRA_PARAMS or BASIC_AUTH environment variable is supplied, the extra_params* parameters will be ignored. Args: user_agent (Optional[str]): User agent string. HDXPythonUtilities/X.X.X- is prefixed. user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied. use_env (bool): Whether to read environment variables. Defaults to True. fail_on_missing_file (bool): Raise an exception if any specified configuration files are missing. Defaults to True. **kwargs: See below auth (Tuple[str, str]): Authorisation information in tuple form (user, pass) OR basic_auth (str): Authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) OR basic_auth_file (str): Path to file containing authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) extra_params_dict (Dict): Extra parameters to put on end of url as a dictionary OR extra_params_json (str): Path to JSON file containing extra parameters to put on end of url OR extra_params_yaml (str): Path to YAML file containing extra parameters to put on end of url extra_params_lookup (str): Lookup key for parameters. If not given assumes parameters are at root of the dict. headers (Dict): Additional headers to add to request. status_forcelist (iterable): HTTP statuses for which to force retry. Defaults to [429, 500, 502, 503, 504]. allowed_methods (iterable): HTTP methods for which to force retry. Defaults t0 frozenset(['GET']). """ s = requests.Session() ua = kwargs.get("full_agent") if not ua: ua = UserAgent.get(user_agent, user_agent_config_yaml, user_agent_lookup, **kwargs) s.headers["User-Agent"] = ua auths_found = list() headers = kwargs.get("headers") if headers is not None: s.headers.update(headers) if "Authorization" in headers: auths_found.append("headers") extra_params_found = False extra_params_dict = None basic_auth = None if use_env: basic_auth_env = os.getenv("BASIC_AUTH") if basic_auth_env: basic_auth = basic_auth_env auths_found.append("basic_auth environment variable") extra_params = os.getenv("EXTRA_PARAMS") if extra_params: if "=" in extra_params: extra_params_dict = dict() logger.info( "Loading extra parameters from environment variable") for extra_param in extra_params.split(","): key, value = extra_param.split("=") extra_params_dict[key] = value extra_params_found = True if not extra_params_found: # only do this if extra params env vars not supplied extra_params_dict = kwargs.get("extra_params_dict") if extra_params_dict: extra_params_found = True logger.info("Loading extra parameters from dictionary") extra_params_json = kwargs.get("extra_params_json", "") if extra_params_json: if extra_params_found: raise SessionError( "More than one set of extra parameters given!") extra_params_found = True logger.info(f"Loading extra parameters from: {extra_params_json}") try: extra_params_dict = load_json(extra_params_json) except OSError: if fail_on_missing_file: raise extra_params_yaml = kwargs.get("extra_params_yaml", "") if extra_params_yaml: if extra_params_found: raise SessionError( "More than one set of extra parameters given!") logger.info(f"Loading extra parameters from: {extra_params_yaml}") try: extra_params_dict = load_yaml(extra_params_yaml) except OSError: if fail_on_missing_file: raise extra_params_lookup = kwargs.get("extra_params_lookup") if extra_params_lookup and extra_params_dict: extra_params_dict = extra_params_dict.get(extra_params_lookup) if extra_params_dict is None: raise SessionError( f"{extra_params_lookup} does not exist in extra_params!") if extra_params_dict: basic_auth_param = extra_params_dict.get("basic_auth") if basic_auth_param: basic_auth = basic_auth_param auths_found.append("basic_auth parameter") del extra_params_dict["basic_auth"] s.params = extra_params_dict basic_auth_arg = kwargs.get("basic_auth") if basic_auth_arg: basic_auth = basic_auth_arg auths_found.append("basic_auth argument") auth = kwargs.get("auth") if auth: auths_found.append("auth argument") basic_auth_file = kwargs.get("basic_auth_file") if basic_auth_file: logger.info(f"Loading basic auth from: {basic_auth_file}") try: basic_auth = load_file_to_str(basic_auth_file, strip=True) auths_found.append(f"file {basic_auth_file}") except OSError: if fail_on_missing_file: raise if len(auths_found) > 1: auths_found_str = ", ".join(auths_found) raise SessionError( f"More than one authorisation given! ({auths_found_str})") if "headers" not in auths_found: if basic_auth: auth = decode(basic_auth) s.auth = auth status_forcelist = kwargs.get("status_forcelist", [429, 500, 502, 503, 504]) allowed_methods = kwargs.get( "allowed_methods", frozenset(["HEAD", "TRACE", "GET", "PUT", "OPTIONS", "DELETE"]), ) retries = Retry( total=5, backoff_factor=0.4, status_forcelist=status_forcelist, allowed_methods=allowed_methods, raise_on_redirect=True, raise_on_status=True, ) s.mount("file://", FileAdapter()) s.mount( "http://", HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100), ) s.mount( "https://", HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100), ) return s
def __init__(self, **kwargs): super(Configuration, self).__init__() hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError('More than one HDX configuration file given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError('More than one HDX configuration file given!') else: if not hdx_config_yaml: logger.info('No HDX configuration parameter. Using default.') hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration) logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError('More than one project configuration file given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError('More than one project configuration file given!') else: if not project_config_yaml: logger.info('No project configuration parameter. Using default.') project_config_yaml = join('config', 'project_configuration.yml') logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict) hdx_key_file = kwargs.get('hdx_key_file', join('%s' % expanduser("~"), '.hdxkey')) self.data['api_key'] = self.load_api_key(hdx_key_file) self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test') if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def test_load_empty(self, fixturesfolder): loaderfolder = join(fixturesfolder, 'loader') with pytest.raises(LoadError): load_yaml(join(loaderfolder, 'empty.yml')) with pytest.raises(LoadError): load_json(join(loaderfolder, 'empty.json'))
def fn(): return load_json( join('tests', 'fixtures', 'ROWCA_movement.json'))
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() self._remoteckan = None self._emailer = None hdx_base_config_found = False hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None) if hdx_base_config_dict: hdx_base_config_found = True logger.info('Loading HDX base configuration from dictionary') hdx_base_config_json = kwargs.get('hdx_base_config_json', '') if hdx_base_config_json: if hdx_base_config_found: raise ConfigurationError( 'More than one HDX base configuration given!') hdx_base_config_found = True logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json) hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '') if hdx_base_config_found: if hdx_base_config_yaml: raise ConfigurationError( 'More than one HDX base configuration given!') else: if not hdx_base_config_yaml: hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info( 'No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml) logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml) hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration given!') else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml if isfile(hdx_config_yaml): logger.info( 'No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml) else: logger.info( 'No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml) hdx_config_yaml = None hdx_config_dict = dict() if hdx_config_yaml: logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get('full_agent') if ua: self.user_agent = ua else: try: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data) self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False)) logger.info('Read only access to HDX: %s' % str(self.hdx_read_only)) self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key')) if not self.hdx_key and not self.hdx_read_only: raise ConfigurationError( 'No HDX API key supplied as a parameter or in configuration!') hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url')) if hdx_url: self.hdx_site = 'hdx_custom_site' hdx_url = hdx_url.rstrip('/') self.data[self.hdx_site] = {'url': hdx_url} else: self.hdx_site = 'hdx_%s_site' % kwargs.get( 'hdx_site', self.data.get('hdx_site', 'test')) if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
"created": "Dec-05-2019", "changed": "Dec-05-2019", "url": "https://microdata.unhcr.org/index.php/catalog/187", }, { "id": "272", "idno": "UNHCR_PHL_2016_Zamboanga_HB_IDP_Profiling", "title": "Zamboanga Home Based IDP Re-Profiling 2016", "nation": "Philippines", "created": "Sep-28-2020", "changed": "Sep-28-2020", "url": "https://microdata.unhcr.org/index.php/catalog/272", }, ], } metadata_187 = load_json(join("tests", "fixtures", "metadata_187.json")) metadata_272 = load_json(join("tests", "fixtures", "metadata_272.json")) class TestUNHCR: @pytest.fixture(scope="function") def configuration(self): Configuration._create( user_agent="test", hdx_key="12345", project_config_yaml=join("tests", "config", "project_configuration.yml"), ) Locations.set_validlocations([ { "name": "afg",
def test_load_json_into_existing_dict(self, configfolder): existing_dict = load_json(join(configfolder, "hdx_config.json")) result = load_json_into_existing_dict( existing_dict, join(configfolder, "project_configuration.json") ) assert list(result.items()) == list(TestLoader.expected_json.items())
def setup_logging(**kwargs: Any) -> None: """Setup logging configuration Args: **kwargs: See below logging_config_dict (dict): Logging configuration dictionary OR logging_config_json (str): Path to JSON Logging configuration OR logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml. smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration Returns: None """ smtp_config_found = False smtp_config_dict = kwargs.get("smtp_config_dict", None) if smtp_config_dict: smtp_config_found = True print("Loading smtp configuration customisations from dictionary") smtp_config_json = kwargs.get("smtp_config_json", "") if smtp_config_json: if smtp_config_found: raise LoggingError("More than one smtp configuration file given!") smtp_config_found = True print( f"Loading smtp configuration customisations from: {smtp_config_json}" ) smtp_config_dict = load_json(smtp_config_json) smtp_config_yaml = kwargs.get("smtp_config_yaml", "") if smtp_config_yaml: if smtp_config_found: raise LoggingError("More than one smtp configuration file given!") smtp_config_found = True print( f"Loading smtp configuration customisations from: {smtp_config_yaml}" ) smtp_config_dict = load_yaml(smtp_config_yaml) logging_smtp_config_dict = None logging_config_found = False logging_config_dict = kwargs.get("logging_config_dict", None) if logging_config_dict: logging_config_found = True print("Loading logging configuration from dictionary") logging_config_json = kwargs.get("logging_config_json", "") if logging_config_json: if logging_config_found: raise LoggingError( "More than one logging configuration file given!") logging_config_found = True print(f"Loading logging configuration from: {logging_config_json}") logging_config_dict = load_json(logging_config_json) logging_config_yaml = kwargs.get("logging_config_yaml", "") if logging_config_found: if logging_config_yaml: raise LoggingError( "More than one logging configuration file given!") else: if not logging_config_yaml: print("No logging configuration parameter. Using default.") logging_config_yaml = script_dir_plus_file( "logging_configuration.yml", setup_logging) if smtp_config_found: logging_smtp_config_yaml = script_dir_plus_file( "logging_smtp_configuration.yml", setup_logging) print( f"Loading base SMTP logging configuration from: {logging_smtp_config_yaml}" ) logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml) print(f"Loading logging configuration from: {logging_config_yaml}") logging_config_dict = load_yaml(logging_config_yaml) if smtp_config_found: if logging_smtp_config_dict: logging_config_dict = merge_dictionaries([ logging_config_dict, logging_smtp_config_dict, smtp_config_dict, ]) else: raise LoggingError( "SMTP logging configuration file given but not using default logging configuration!" ) file_only = os.getenv("LOG_FILE_ONLY") if file_only is not None and file_only.lower() not in [ "false", "f", "n", "no", "0", ]: root = logging_config_dict.get("root") if root is not None: handlers = root.get("handlers", list()) for i, handler in enumerate(handlers): if handler.lower() == "console": del handlers[i] break logging.config.dictConfig(logging_config_dict)
def test_load_json_into_existing_dict(self, configfolder): existing_dict = load_json(join(configfolder, 'hdx_config.json')) result = load_json_into_existing_dict( existing_dict, join(configfolder, 'project_configuration.json')) assert result == TestLoader.expected_json
def fn(): return load_json( join('tests', 'fixtures', 'FTS_plan_NGA.json'))
def fn(): return load_json( join('tests', 'fixtures', 'CBPF_Location_SSD.json'))
def fn(): return load_json( join('tests', 'fixtures', 'CBPF_ProjectSummary_SSD.json'))
""" from os.path import join import hdx import pytest from hdx.data.vocabulary import Vocabulary from hdx.api.configuration import Configuration from hdx.api.locations import Locations from hdx.location.country import Country from hdx.utilities.uuid import is_valid_uuid from hdx.utilities.loader import load_json from hdx.utilities.errors_onexit import ErrorsOnExit from cods import COD alljson = load_json(join("tests", "fixtures", "apiinput.json")) class TestCods: @pytest.fixture(scope="function") def configuration(self): Configuration._create( user_agent="test", hdx_key="12345", project_config_yaml=join("tests", "config", "project_configuration.yml"), ) Locations.set_validlocations([ { "name": "afg", "title": "Afghanistan"
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() self._remoteckan = None self._emailer = None hdx_base_config_found = False hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None) if hdx_base_config_dict: hdx_base_config_found = True logger.info('Loading HDX base configuration from dictionary') hdx_base_config_json = kwargs.get('hdx_base_config_json', '') if hdx_base_config_json: if hdx_base_config_found: raise ConfigurationError('More than one HDX base configuration given!') hdx_base_config_found = True logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json) hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '') if hdx_base_config_found: if hdx_base_config_yaml: raise ConfigurationError('More than one HDX base configuration given!') else: if not hdx_base_config_yaml: hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info('No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml) logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml) hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError('More than one HDX configuration given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError('More than one HDX configuration given!') else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml if isfile(hdx_config_yaml): logger.info('No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml) else: logger.info('No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml) hdx_config_yaml = None hdx_config_dict = dict() if hdx_config_yaml: logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError('More than one project configuration given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError('More than one project configuration given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get('full_agent') if ua: self.user_agent = ua else: try: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data) self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False)) logger.info('Read only access to HDX: %s' % str(self.hdx_read_only)) self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key')) if not self.hdx_key and not self.hdx_read_only: raise ConfigurationError('No HDX API key supplied as a parameter or in configuration!') hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url')) if hdx_url: self.hdx_site = 'hdx_custom_site' self.data[self.hdx_site] = {'url': hdx_url} else: self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', self.data.get('hdx_site', 'test')) if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def setup_logging(**kwargs) -> None: """Setup logging configuration Args: **kwargs: See below logging_config_dict (dict): Logging configuration dictionary OR logging_config_json (str): Path to JSON Logging configuration OR logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml. smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration Returns: None """ smtp_config_found = False smtp_config_dict = kwargs.get('smtp_config_dict', None) if smtp_config_dict: smtp_config_found = True print('Loading smtp configuration customisations from dictionary') smtp_config_json = kwargs.get('smtp_config_json', '') if smtp_config_json: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_json) smtp_config_dict = load_json(smtp_config_json) smtp_config_yaml = kwargs.get('smtp_config_yaml', '') if smtp_config_yaml: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_yaml) smtp_config_dict = load_yaml(smtp_config_yaml) logging_smtp_config_dict = None logging_config_found = False logging_config_dict = kwargs.get('logging_config_dict', None) if logging_config_dict: logging_config_found = True print('Loading logging configuration from dictionary') logging_config_json = kwargs.get('logging_config_json', '') if logging_config_json: if logging_config_found: raise LoggingError('More than one logging configuration file given!') logging_config_found = True print('Loading logging configuration from: %s' % logging_config_json) logging_config_dict = load_json(logging_config_json) logging_config_yaml = kwargs.get('logging_config_yaml', '') if logging_config_found: if logging_config_yaml: raise LoggingError('More than one logging configuration file given!') else: if not logging_config_yaml: print('No logging configuration parameter. Using default.') logging_config_yaml = script_dir_plus_file('logging_configuration.yml', setup_logging) if smtp_config_found: logging_smtp_config_yaml = script_dir_plus_file('logging_smtp_configuration.yml', setup_logging) print('Loading base SMTP logging configuration from: %s' % logging_smtp_config_yaml) logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml) print('Loading logging configuration from: %s' % logging_config_yaml) logging_config_dict = load_yaml(logging_config_yaml) if smtp_config_found: if logging_smtp_config_dict: logging_config_dict = merge_dictionaries([logging_config_dict, logging_smtp_config_dict, smtp_config_dict]) else: raise LoggingError('SMTP logging configuration file given but not using default logging configuration!') logging.config.dictConfig(logging_config_dict)
def get_session(**kwargs): # type: (Any) -> requests.Session """Set up and return Session object that is set up with retrying Args: **kwargs: See below auth (Tuple[str, str]): Authorisation information in tuple form (user, pass) OR basic_auth (str): Authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) OR basic_auth_file (str): Path to file containing authorisation information in basic auth string form (Basic xxxxxxxxxxxxxxxx) extra_params_dict (Dict): Extra parameters to put on end of url as a dictionary OR extra_params_json (str): Path to JSON file containing extra parameters to put on end of url OR extra_params_yaml (str): Path to YAML file containing extra parameters to put on end of url extra_params_lookup (str): Lookup key for parameters. If not given assumes parameters are at root of the dict. status_forcelist (iterable): HTTP statuses for which to force retry. Defaults to [429, 500, 502, 503, 504]. method_whitelist (iterable): HTTP methods for which to force retry. Defaults t0 frozenset(['GET']). """ s = requests.Session() extra_params_found = False extra_params_dict = kwargs.get('extra_params_dict', None) if extra_params_dict: extra_params_found = True logger.info('Loading extra parameters from dictionary') extra_params_json = kwargs.get('extra_params_json', '') if extra_params_json: if extra_params_found: raise SessionError('More than one set of extra parameters given!') extra_params_found = True logger.info('Loading extra parameters from: %s' % extra_params_json) extra_params_dict = load_json(extra_params_json) extra_params_yaml = kwargs.get('extra_params_yaml', '') if extra_params_found: if extra_params_yaml: raise SessionError('More than one set of extra parameters given!') else: if extra_params_yaml: logger.info('Loading extra parameters from: %s' % extra_params_yaml) extra_params_dict = load_yaml(extra_params_yaml) else: extra_params_dict = dict() extra_params_lookup = kwargs.get('extra_params_lookup') if extra_params_lookup: extra_params_dict = extra_params_dict.get(extra_params_lookup) if extra_params_dict is None: raise SessionError('%s does not exist in extra_params!' % extra_params_lookup) auth_found = False basic_auth = extra_params_dict.get('basic_auth') if basic_auth: logger.info('Loading authorisation from basic_auth parameter') auth_found = True del extra_params_dict['basic_auth'] s.params = extra_params_dict auth = kwargs.get('auth') if auth: if auth_found: raise SessionError('More than one authorisation given!') logger.info('Loading authorisation from auth argument') auth_found = True bauth = kwargs.get('basic_auth') if bauth: if auth_found: raise SessionError('More than one authorisation given!') logger.info('Loading authorisation from basic_auth argument') basic_auth = bauth auth_found = True basic_auth_file = kwargs.get('basic_auth_file') if basic_auth_file: if auth_found: raise SessionError('More than one authorisation given!') logger.info('Loading authorisation from: %s' % basic_auth_file) basic_auth = load_file_to_str(basic_auth_file) if basic_auth: auth = decode(basic_auth) s.auth = auth status_forcelist = kwargs.get('status_forcelist', [429, 500, 502, 503, 504]) method_whitelist = kwargs.get( 'method_whitelist', frozenset(['HEAD', 'TRACE', 'GET', 'PUT', 'OPTIONS', 'DELETE'])) retries = Retry(total=5, backoff_factor=0.4, status_forcelist=status_forcelist, method_whitelist=method_whitelist, raise_on_redirect=True, raise_on_status=True) s.mount( 'http://', HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100)) s.mount( 'https://', HTTPAdapter(max_retries=retries, pool_connections=100, pool_maxsize=100)) return s
def fn(): return load_json( join('tests', 'fixtures', 'ROWCA_population.json'))