def countriesdata(cls, use_live=True): # type: (bool) -> List[Dict[Dict]] """ Read countries data from OCHA countries feed (falling back to file) Args: use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True. Returns: List[Dict[Dict]]: Countries dictionaries """ if cls._countriesdata is None: countries = None if use_live: try: countries = hxl.data(cls._ochaurl) except IOError: logger.exception( 'Download from OCHA feed failed! Falling back to stored file.' ) if countries is None: countries = hxl.data(script_dir_plus_file( 'Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv', Country), allow_local=True) cls.set_countriesdata(countries) return cls._countriesdata
def __init__(self, configuration: Configuration, initial_data: dict): super(HDXObject, self).__init__(initial_data) self.configuration = configuration self.old_data = None version_file = open(script_dir_plus_file(join('..', 'version.txt'), HDXObject)) version = version_file.read().strip() self.hdxpostsite = ckanapi.RemoteCKAN(configuration.get_hdx_site_url(), apikey=configuration.get_api_key(), user_agent='HDXPythonLibrary/%s' % version)
def test_ocha_feed_file_working(self): countries = hxl.data(script_dir_plus_file('Countries_UZB_Deleted.csv', TestCountry), allow_local=True) Country.set_countriesdata(countries) assert Country.get_iso3_country_code('UZBEKISTAN') is None assert Country.get_iso3_country_code('south sudan') == 'SSD' Country.set_ocha_url() Country._countriesdata = None assert Country.get_iso3_country_code('UZBEKISTAN', use_live=True) == 'UZB' Country.set_ocha_url('NOTEXIST') Country._countriesdata = None assert Country.get_iso3_from_iso2('AF') == 'AFG'
def create_datastore_for_topline(self, delete_first: int = 0): """Creates a resource in the HDX datastore using the built in YAML definition for a topline Args: delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. Returns: None """ data = load_yaml( script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource)) self.create_datastore_from_dict_schema(data, delete_first)
def create_remoteckan(self): # type: () -> ckanapi.RemoteCKAN """ Create remote CKAN instance from configuration Returns: ckanapi.RemoteCKAN: Remote CKAN instance """ version_file = open(script_dir_plus_file('version.txt', Configuration)) version = version_file.read().strip() return ckanapi.RemoteCKAN(self.get_hdx_site_url(), apikey=self.get_api_key(), user_agent='HDXPythonLibrary/%s' % version)
def create_datastore_for_topline(self, delete_first=0, path=None): # type: (int, Optional[str]) -> None """For tabular data, create a resource in the HDX datastore which enables data preview in HDX using the built in YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX. Args: delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. path (Optional[str]): Local path to file that was uploaded. Defaults to None. Returns: None """ data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource)) self.create_datastore_from_dict_schema(data, delete_first, path=path)
def main(hdx_key, user_agent, preprefix, hdx_site, db_url, save): project_config_yaml = script_dir_plus_file('project_configuration.yml', main) site_url = Configuration.create(hdx_key=hdx_key, hdx_site=hdx_site, user_agent=user_agent, preprefix=preprefix, project_config_yaml=project_config_yaml) logger.info('--------------------------------------------------') logger.info('> HDX Site: %s' % site_url) if db_url: logger.info('> DB URL: %s' % db_url) if 'postgres' in db_url: result = urlparse(db_url) username = result.username password = result.password database = result.path[1:] hostname = result.hostname connecting_string = 'Checking for PostgreSQL...' while True: try: logger.info(connecting_string) connection = psycopg2.connect(database=database, user=username, password=password, host=hostname, connect_timeout=3) connection.close() logger.info('PostgreSQL is running!') break except psycopg2.OperationalError: time.sleep(1) testsession = None if save: engine = create_engine('sqlite:///test_serialize.db', poolclass=NullPool, echo=False) Session = sessionmaker(bind=engine) TestBase.metadata.create_all(engine) testsession = Session() freshness = DataFreshness(db_url=db_url, testsession=testsession) datasets_to_check, resources_to_check = freshness.process_datasets() results, hash_results = freshness.check_urls(resources_to_check) datasets_lastmodified = freshness.process_results(results, hash_results) freshness.update_dataset_last_modified(datasets_to_check, datasets_lastmodified) freshness.output_counts() freshness.close() logger.info('Freshness completed!')
def create_datastore_for_topline(self, delete_first=0, path=None): # type: (int, Optional[str]) -> None """For csvs, create a resource in the HDX datastore which enables data preview in HDX using the built in YAML definition for a topline. If path is not supplied, the file is first downloaded from HDX. Args: delete_first (int): Delete datastore before creation. 0 = No, 1 = Yes, 2 = If no primary key. Defaults to 0. path (Optional[str]): Local path to file that was uploaded. Defaults to None. Returns: None """ data = load_yaml(script_dir_plus_file(join('..', 'hdx_datasource_topline.yml'), Resource)) self.create_datastore_from_dict_schema(data, delete_first, path=path)
def create(**kwargs) -> 'Configuration': """ Create Args: path (str): Path to HDX key Returns: str: HDX api key """ hdx.configuration = Configuration(**kwargs) version_file = open(script_dir_plus_file('version.txt', Configuration)) version = version_file.read().strip() hdx.remoteckan = ckanapi.RemoteCKAN( hdx.configuration.get_hdx_site_url(), apikey=hdx.configuration.get_api_key(), user_agent='HDXPythonLibrary/%s' % version) return hdx.configuration
def create_remoteckan( self, session=get_session(method_whitelist=frozenset( ['HEAD', 'TRACE', 'GET', 'POST', 'PUT', 'OPTIONS', 'DELETE']))): # type: () -> ckanapi.RemoteCKAN """ Create remote CKAN instance from configuration Args: session: requests Session object to use. Defaults to calling hdx.utilities.session.get_session() Returns: ckanapi.RemoteCKAN: Remote CKAN instance """ version_file = open(script_dir_plus_file('version.txt', Configuration)) version = version_file.read().strip() return ckanapi.RemoteCKAN(self.get_hdx_site_url(), session=session, user_agent='HDXPythonLibrary/%s' % version)
def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration file given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration file given!') else: if not hdx_config_yaml: logger.info('No HDX configuration parameter. Using default.') hdx_config_yaml = script_dir_plus_file('hdx_configuration.yml', Configuration) logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration file given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration file given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_config_dict, project_config_dict) self.hdx_read_only = kwargs.get('hdx_read_only', False) if not self.hdx_read_only: if 'hdx_key' in kwargs: self.data['api_key'] = kwargs.get('hdx_key') else: hdx_key_file = kwargs.get('hdx_key_file', join(expanduser('~'), '.hdxkey')) """ :type : str""" self.data['api_key'] = self.load_api_key(hdx_key_file) self.hdx_site = 'hdx_%s_site' % kwargs.get('hdx_site', 'test') if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site)
def setup_logging(**kwargs) -> None: """Setup logging configuration Args: **kwargs: See below logging_config_dict (dict): Logging configuration dictionary OR logging_config_json (str): Path to JSON Logging configuration OR logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml. smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration Returns: None """ smtp_config_found = False smtp_config_dict = kwargs.get('smtp_config_dict', None) if smtp_config_dict: smtp_config_found = True print('Loading smtp configuration customisations from dictionary') smtp_config_json = kwargs.get('smtp_config_json', '') if smtp_config_json: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_json) smtp_config_dict = load_json(smtp_config_json) smtp_config_yaml = kwargs.get('smtp_config_yaml', '') if smtp_config_yaml: if smtp_config_found: raise LoggingError('More than one smtp configuration file given!') smtp_config_found = True print('Loading smtp configuration customisations from: %s' % smtp_config_yaml) smtp_config_dict = load_yaml(smtp_config_yaml) logging_smtp_config_dict = None logging_config_found = False logging_config_dict = kwargs.get('logging_config_dict', None) if logging_config_dict: logging_config_found = True print('Loading logging configuration from dictionary') logging_config_json = kwargs.get('logging_config_json', '') if logging_config_json: if logging_config_found: raise LoggingError( 'More than one logging configuration file given!') logging_config_found = True print('Loading logging configuration from: %s' % logging_config_json) logging_config_dict = load_json(logging_config_json) logging_config_yaml = kwargs.get('logging_config_yaml', '') if logging_config_found: if logging_config_yaml: raise LoggingError( 'More than one logging configuration file given!') else: if not logging_config_yaml: print('No logging configuration parameter. Using default.') logging_config_yaml = script_dir_plus_file( 'logging_configuration.yml', setup_logging) if smtp_config_found: logging_smtp_config_yaml = script_dir_plus_file( 'logging_smtp_configuration.yml', setup_logging) print('Loading base SMTP logging configuration from: %s' % logging_smtp_config_yaml) logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml) print('Loading logging configuration from: %s' % logging_config_yaml) logging_config_dict = load_yaml(logging_config_yaml) if smtp_config_found: if logging_smtp_config_dict: logging_config_dict = merge_dictionaries([ logging_config_dict, logging_smtp_config_dict, smtp_config_dict ]) else: raise LoggingError( 'SMTP logging configuration file given but not using default logging configuration!' ) logging.config.dictConfig(logging_config_dict)
def setup_logging(**kwargs: Any) -> None: """Setup logging configuration Args: **kwargs: See below logging_config_dict (dict): Logging configuration dictionary OR logging_config_json (str): Path to JSON Logging configuration OR logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml. smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration Returns: None """ smtp_config_found = False smtp_config_dict = kwargs.get("smtp_config_dict", None) if smtp_config_dict: smtp_config_found = True print("Loading smtp configuration customisations from dictionary") smtp_config_json = kwargs.get("smtp_config_json", "") if smtp_config_json: if smtp_config_found: raise LoggingError("More than one smtp configuration file given!") smtp_config_found = True print( f"Loading smtp configuration customisations from: {smtp_config_json}" ) smtp_config_dict = load_json(smtp_config_json) smtp_config_yaml = kwargs.get("smtp_config_yaml", "") if smtp_config_yaml: if smtp_config_found: raise LoggingError("More than one smtp configuration file given!") smtp_config_found = True print( f"Loading smtp configuration customisations from: {smtp_config_yaml}" ) smtp_config_dict = load_yaml(smtp_config_yaml) logging_smtp_config_dict = None logging_config_found = False logging_config_dict = kwargs.get("logging_config_dict", None) if logging_config_dict: logging_config_found = True print("Loading logging configuration from dictionary") logging_config_json = kwargs.get("logging_config_json", "") if logging_config_json: if logging_config_found: raise LoggingError( "More than one logging configuration file given!") logging_config_found = True print(f"Loading logging configuration from: {logging_config_json}") logging_config_dict = load_json(logging_config_json) logging_config_yaml = kwargs.get("logging_config_yaml", "") if logging_config_found: if logging_config_yaml: raise LoggingError( "More than one logging configuration file given!") else: if not logging_config_yaml: print("No logging configuration parameter. Using default.") logging_config_yaml = script_dir_plus_file( "logging_configuration.yml", setup_logging) if smtp_config_found: logging_smtp_config_yaml = script_dir_plus_file( "logging_smtp_configuration.yml", setup_logging) print( f"Loading base SMTP logging configuration from: {logging_smtp_config_yaml}" ) logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml) print(f"Loading logging configuration from: {logging_config_yaml}") logging_config_dict = load_yaml(logging_config_yaml) if smtp_config_found: if logging_smtp_config_dict: logging_config_dict = merge_dictionaries([ logging_config_dict, logging_smtp_config_dict, smtp_config_dict, ]) else: raise LoggingError( "SMTP logging configuration file given but not using default logging configuration!" ) file_only = os.getenv("LOG_FILE_ONLY") if file_only is not None and file_only.lower() not in [ "false", "f", "n", "no", "0", ]: root = logging_config_dict.get("root") if root is not None: handlers = root.get("handlers", list()) for i, handler in enumerate(handlers): if handler.lower() == "console": del handlers[i] break logging.config.dictConfig(logging_config_dict)
db_url = getenv("DB_URL") if db_url and "://" not in db_url: db_url = f"postgresql://{db_url}" gsheet_auth = args.gsheet_auth if gsheet_auth is None: gsheet_auth = getenv("GSHEET_AUTH") email_server = args.email_server if email_server is None: email_server = getenv("EMAIL_SERVER") failure_emails = args.failure_emails if failure_emails is None: failure_emails = getenv("FAILURE_EMAILS") sysadmin_emails = args.sysadmin_emails if sysadmin_emails is None: sysadmin_emails = getenv("SYSADMIN_EMAILS") project_config_yaml = script_dir_plus_file("project_configuration.yml", main) facade( main, hdx_key=hdx_key, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=project_config_yaml, db_url=db_url, db_params=args.db_params, gsheet_auth=gsheet_auth, email_server=email_server, failure_emails=failure_emails, sysadmin_emails=sysadmin_emails, email_test=args.email_test, spreadsheet_test=args.spreadsheet_test,
def get_country_version(): version_file = open(script_dir_plus_file('version.txt', get_country_version)) return version_file.read().strip()
class Configuration(UserDict, object): """Configuration for HDX Args: **kwargs: See below user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not. user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied. hdx_url (str): HDX url to use. Overrides hdx_site. hdx_site (str): HDX site to use eg. prod, test. hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False. hdx_key (str): Your HDX key. Ignored if hdx_read_only = True. hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR hdx_config_json (str): Path to JSON HDX configuration OR hdx_config_yaml (str): Path to YAML HDX configuration project_config_dict (dict): Project configuration dictionary OR project_config_json (str): Path to JSON Project configuration OR project_config_yaml (str): Path to YAML Project configuration hdx_base_config_dict (dict): HDX base configuration dictionary OR hdx_base_config_json (str): Path to JSON HDX base configuration OR hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml. """ _configuration = None home_folder = expanduser('~') default_hdx_base_config_yaml = script_dir_plus_file( 'hdx_base_configuration.yml', ConfigurationError) default_hdx_config_yaml = join(home_folder, '.hdx_configuration.yml') apiversion = get_api_version() prefix = 'HDXPythonLibrary/%s' % apiversion def __init__(self, **kwargs): # type: (Any) -> None super(Configuration, self).__init__() self._remoteckan = None self._emailer = None hdx_base_config_found = False hdx_base_config_dict = kwargs.get('hdx_base_config_dict', None) if hdx_base_config_dict: hdx_base_config_found = True logger.info('Loading HDX base configuration from dictionary') hdx_base_config_json = kwargs.get('hdx_base_config_json', '') if hdx_base_config_json: if hdx_base_config_found: raise ConfigurationError( 'More than one HDX base configuration given!') hdx_base_config_found = True logger.info('Loading HDX base configuration from: %s' % hdx_base_config_json) hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get('hdx_base_config_yaml', '') if hdx_base_config_found: if hdx_base_config_yaml: raise ConfigurationError( 'More than one HDX base configuration given!') else: if not hdx_base_config_yaml: hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info( 'No HDX base configuration parameter. Using default base configuration file: %s.' % hdx_base_config_yaml) logger.info('Loading HDX base configuration from: %s' % hdx_base_config_yaml) hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False hdx_config_dict = kwargs.get('hdx_config_dict', None) if hdx_config_dict: hdx_config_found = True logger.info('Loading HDX configuration from dictionary') hdx_config_json = kwargs.get('hdx_config_json', '') if hdx_config_json: if hdx_config_found: raise ConfigurationError( 'More than one HDX configuration given!') hdx_config_found = True logger.info('Loading HDX configuration from: %s' % hdx_config_json) hdx_config_dict = load_json(hdx_config_json) hdx_config_yaml = kwargs.get('hdx_config_yaml', '') if hdx_config_found: if hdx_config_yaml: raise ConfigurationError( 'More than one HDX configuration given!') else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml if isfile(hdx_config_yaml): logger.info( 'No HDX configuration parameter. Using default configuration file: %s.' % hdx_config_yaml) else: logger.info( 'No HDX configuration parameter and no configuration file at default path: %s.' % hdx_config_yaml) hdx_config_yaml = None hdx_config_dict = dict() if hdx_config_yaml: logger.info('Loading HDX configuration from: %s' % hdx_config_yaml) hdx_config_dict = load_yaml(hdx_config_yaml) self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get('project_config_dict', None) if project_config_dict is not None: project_config_found = True logger.info('Loading project configuration from dictionary') project_config_json = kwargs.get('project_config_json', '') if project_config_json: if project_config_found: raise ConfigurationError( 'More than one project configuration given!') project_config_found = True logger.info('Loading project configuration from: %s' % project_config_json) project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get('project_config_yaml', '') if project_config_found: if project_config_yaml: raise ConfigurationError( 'More than one project configuration given!') else: if project_config_yaml: logger.info('Loading project configuration from: %s' % project_config_yaml) project_config_dict = load_yaml(project_config_yaml) else: project_config_dict = dict() self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get('full_agent') if ua: self.user_agent = ua else: try: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get(prefix=Configuration.prefix, **self.data) self.hdx_read_only = kwargs.get('hdx_read_only', self.data.get('hdx_read_only', False)) logger.info('Read only access to HDX: %s' % str(self.hdx_read_only)) self.hdx_key = kwargs.get('hdx_key', self.data.get('hdx_key')) if not self.hdx_key and not self.hdx_read_only: raise ConfigurationError( 'No HDX API key supplied as a parameter or in configuration!') hdx_url = kwargs.get('hdx_url', self.data.get('hdx_url')) if hdx_url: self.hdx_site = 'hdx_custom_site' hdx_url = hdx_url.rstrip('/') self.data[self.hdx_site] = {'url': hdx_url} else: self.hdx_site = 'hdx_%s_site' % kwargs.get( 'hdx_site', self.data.get('hdx_site', 'test')) if self.hdx_site not in self.data: raise ConfigurationError('%s not defined in configuration!' % self.hdx_site) def set_read_only(self, read_only=True): # type: (bool) -> None """ Set HDX read only flag Args: read_only (bool): Value to set HDX read only flag. Defaults to True. Returns: None """ self.hdx_read_only = read_only def set_api_key(self, apikey): # type: (str) -> None """ Set HDX api key Args: apikey (str): Value to set api key. Returns: None """ self.hdx_key = apikey def get_api_key(self): # type: () -> Optional[str] """ Return HDX api key or None if read only Returns: Optional[str]: HDX api key or None if read only """ if self.hdx_read_only: return None return self.hdx_key def get_user_agent(self): # type: () -> str """ Return user agent Returns: str: User agent """ return self.user_agent def get_hdx_site_url(self): # type: () -> str """ Return HDX web site url Returns: str: HDX web site url """ return self.data[self.hdx_site]['url'] def get_dataset_url(self, name): # type: (str) -> str """ Return HDX dataset url given dataset name Args: name: Dataset name Returns: str: HDX dataset url """ return '%s/dataset/%s' % (self.get_hdx_site_url(), name) def _get_credentials(self): # type: () -> Optional[Tuple[str, str]] """ Return HDX site username and password Returns: Optional[Tuple[str, str]]: HDX site username and password or None """ site = self.data[self.hdx_site] username = site.get('username') if username: return b64decode(username).decode('utf-8'), b64decode( site['password']).decode('utf-8') else: return None def remoteckan(self): # type: () -> ckanapi.RemoteCKAN """ Return the remote CKAN object (see ckanapi library) Returns: ckanapi.RemoteCKAN: The remote CKAN object """ if self._remoteckan is None: raise ConfigurationError( 'There is no remote CKAN set up! Use Configuration.create(**kwargs)' ) return self._remoteckan def call_remoteckan(self, *args, **kwargs): # type: (Any, Any) -> Dict """ Calls the remote CKAN Args: *args: Arguments to pass to remote CKAN call_action method **kwargs: Keyword arguments to pass to remote CKAN call_action method Returns: Dict: The response from the remote CKAN call_action method """ requests_kwargs = kwargs.get('requests_kwargs', dict()) credentials = self._get_credentials() if credentials: requests_kwargs['auth'] = credentials kwargs['requests_kwargs'] = requests_kwargs apikey = kwargs.get('apikey', self.get_api_key()) kwargs['apikey'] = apikey return self.remoteckan().call_action(*args, **kwargs) @staticmethod def _environment_variables(**kwargs): # type: (Any) -> Any """ Overwrite keyword arguments with environment variables Args: **kwargs: See below hdx_url (str): HDX url to use. Overrides hdx_site. hdx_site (str): HDX site to use eg. prod, test. Defaults to test. hdx_key (str): Your HDX key. Ignored if hdx_read_only = True. Returns: kwargs: Changed keyword arguments """ hdx_key = os.getenv('HDX_KEY') if hdx_key is not None: kwargs['hdx_key'] = hdx_key hdx_url = os.getenv('HDX_URL') if hdx_url is not None: kwargs['hdx_url'] = hdx_url else: hdx_site = os.getenv('HDX_SITE') if hdx_site is not None: kwargs['hdx_site'] = hdx_site return kwargs @classmethod def create_remoteckan(cls, site_url, user_agent=None, user_agent_config_yaml=None, user_agent_lookup=None, use_env=False, session=None, **kwargs): # type: (str, Optional[str], Optional[str], Optional[str], bool, requests.Session, Any) -> ckanapi.RemoteCKAN """ Create remote CKAN instance from configuration Args: site_url (str): Site url. user_agent (Optional[str]): User agent string. HDXPythonLibrary/X.X.X- is prefixed. user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied. use_env (bool): Whether to read environment variables. Defaults to False. session (requests.Session): requests Session object to use. Defaults to calling hdx.utilities.session.get_session() Returns: ckanapi.RemoteCKAN: Remote CKAN instance """ if not session: whitelist = frozenset( ['HEAD', 'TRACE', 'GET', 'POST', 'PUT', 'OPTIONS', 'DELETE']) session = get_session(user_agent, user_agent_config_yaml, user_agent_lookup, use_env, prefix=Configuration.prefix, method_whitelist=whitelist, **kwargs) ua = session.headers['User-Agent'] else: ua = kwargs.get('full_agent') if not ua: ua = UserAgent.get(user_agent, user_agent_config_yaml, user_agent_lookup, prefix=Configuration.prefix, **kwargs) return ckanapi.RemoteCKAN(site_url, user_agent=ua, session=session) def setup_remoteckan(self, remoteckan=None, **kwargs): # type: (Optional[ckanapi.RemoteCKAN], Any) -> None """ Set up remote CKAN from provided CKAN or by creating from configuration Args: remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration. Returns: None """ if remoteckan is None: self._remoteckan = self.create_remoteckan( self.get_hdx_site_url(), full_agent=self.get_user_agent(), **kwargs) else: self._remoteckan = remoteckan def emailer(self): # type: () -> Email """ Return the Email object (see :any:`Email`) Returns: Email: The email object """ if self._emailer is None: raise ConfigurationError( 'There is no emailer set up! Use setup_emailer(Any)') return self._emailer def setup_emailer(self, **kwargs): # type: (Any) -> None """ Set up emailer. Parameters in dictionary or file (eg. yaml below): | connection_type: "ssl" ("ssl" for smtp ssl or "lmtp", otherwise basic smtp is assumed) | host: "localhost" | port: 123 | local_hostname: "mycomputer.fqdn.com" | timeout: 3 | username: "******" | password: "******" Args: **kwargs: See below email_config_dict (dict): HDX configuration dictionary OR email_config_json (str): Path to JSON HDX configuration OR email_config_yaml (str): Path to YAML HDX configuration. Defaults to ~/hdx_email_configuration.yml. Returns: None """ self._emailer = Email(**kwargs) @classmethod def read(cls): # type: () -> 'Configuration' """ Read the HDX configuration Returns: Configuration: The HDX configuration """ if cls._configuration is None: raise ConfigurationError( 'There is no HDX configuration! Use Configuration.create(**kwargs)' ) return cls._configuration @classmethod def setup(cls, configuration=None, **kwargs): # type: (Optional['Configuration'], Any) -> None """ Set up the HDX configuration Args: configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments. **kwargs: See below user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not. user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied. hdx_url (str): HDX url to use. Overrides hdx_site. hdx_site (str): HDX site to use eg. prod, test. hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False. hdx_key (str): Your HDX key. Ignored if hdx_read_only = True. hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR hdx_config_json (str): Path to JSON HDX configuration OR hdx_config_yaml (str): Path to YAML HDX configuration project_config_dict (dict): Project configuration dictionary OR project_config_json (str): Path to JSON Project configuration OR project_config_yaml (str): Path to YAML Project configuration hdx_base_config_dict (dict): HDX base configuration dictionary OR hdx_base_config_json (str): Path to JSON HDX base configuration OR hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml. Returns: None """ if configuration is None: cls._configuration = Configuration(**kwargs) else: cls._configuration = configuration @classmethod def _create(cls, configuration=None, remoteckan=None, **kwargs): # type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str """ Create HDX configuration Args: configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments. remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration. **kwargs: See below user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not. user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied. hdx_url (str): HDX url to use. Overrides hdx_site. hdx_site (str): HDX site to use eg. prod, test. hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False. hdx_key (str): Your HDX key. Ignored if hdx_read_only = True. hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR hdx_config_json (str): Path to JSON HDX configuration OR hdx_config_yaml (str): Path to YAML HDX configuration project_config_dict (dict): Project configuration dictionary OR project_config_json (str): Path to JSON Project configuration OR project_config_yaml (str): Path to YAML Project configuration hdx_base_config_dict (dict): HDX base configuration dictionary OR hdx_base_config_json (str): Path to JSON HDX base configuration OR hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml. Returns: str: HDX site url """ kwargs = cls._environment_variables(**kwargs) cls.setup(configuration, **kwargs) cls._configuration.setup_remoteckan(remoteckan, **kwargs) return cls._configuration.get_hdx_site_url() @classmethod def create(cls, configuration=None, remoteckan=None, **kwargs): # type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str """ Create HDX configuration. Can only be called once (will raise an error if called more than once). Args: configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments. remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration. **kwargs: See below user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not. user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml. user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied. hdx_url (str): HDX url to use. Overrides hdx_site. hdx_site (str): HDX site to use eg. prod, test. hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False. hdx_key (str): Your HDX key. Ignored if hdx_read_only = True. hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR hdx_config_json (str): Path to JSON HDX configuration OR hdx_config_yaml (str): Path to YAML HDX configuration project_config_dict (dict): Project configuration dictionary OR project_config_json (str): Path to JSON Project configuration OR project_config_yaml (str): Path to YAML Project configuration hdx_base_config_dict (dict): HDX base configuration dictionary OR hdx_base_config_json (str): Path to JSON HDX base configuration OR hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml. Returns: str: HDX site url """ if cls._configuration is not None: raise ConfigurationError('Configuration already created!') return cls._create(configuration=configuration, remoteckan=remoteckan, **kwargs) @classmethod def delete(cls): # type: () -> None """ Delete global HDX configuration. Returns: None """ cls._configuration = None
def get_api_version(): version_file = open(script_dir_plus_file('version.txt', get_api_version)) return version_file.read().strip()