def _read_config(self): """Reads the config file with the Rebrandly API key""" import os file_path = os.path.join(os.path.dirname(__file__), './config.ini') config = ConfigParser() config.read(file_path) self._apikey = config.get('Rebrandly', 'apikey')
def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False
def __init__(self, default_config=None, *args, **kwargs): super(XToolConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) # 读取配置字符串 if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False
def __init__(self, default_config=None, *args, **kwargs): super().__init__(*args, **kwargs) self.excalibur_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.excalibur_defaults.read_string(default_config) self.is_validated = False
def __init__(self, filename): ConfigParser.__init__(self) # let's read the file if isinstance(filename, string_types): self.filename = filename self.read(filename) else: # pragma: no cover self.filename = None self.read_file(filename)
def __init__(self, lcfname): ''' Constructor ''' super(LayerFileReader, self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename)
def _populate_config(self) -> ConfigParser: config = ConfigParser() config_path = './resources/config.ini' if 'resources' not in listdir('.'): config_path = '.' + config_path with open(config_path) as file_pointer: config.read_file(file_pointer) return config
def __init__(self, config_name="flexipy/flexipy.conf"): self.conf = ConfigParser() #use resource management api to find flexipy.conf, see docs filename = resource_filename(Requirement.parse("flexipy"), config_name) # Open the file with the correct encoding try: with codecs.open(filename, 'r', encoding='utf-8') as f: self.conf.readfp(f) except IOError: raise ValueError('Konfiguracni soubor ' + config_name + ' neexistuje nebo jste uvedli spatnou cestu.')
def __init__(self): ''' Constructor ''' self.dvalue = None self.dselect = 'dest' #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02 self.plist = ('lgvalue', 'uconf', 'epsg', 'fd', 'td') self.cp = ConfigParser() self.fn = os.path.join(os.path.dirname(__file__), self.GUI_PREFS) with codecs.open(self.fn, 'r', 'utf-8') as cf: self.cp.readfp(cf)
def setUpClass(cls): cls.config = ConfigParser() cls.config.read([ r'pam_mysql_manager-test.conf', r'/etc/pam_mysql_manager-test.conf', os.path.expanduser('~/.pam_mysql_manager-test.conf') ]) mysql_user = os.getenv( "PAMMYSQL_TEST_MYSQL_USER", cls.config.get('database', 'user', fallback='root')) mysql_pass = os.getenv( "PAMMYSQL_TEST_MYSQL_PASS", cls.config.get('database', 'password', fallback='')) mysql_host = os.getenv( "PAMMYSQL_TEST_MYSQL_HOST", cls.config.get('database', 'host', fallback='localhost')) mysql_port = int( os.getenv("PAMMYSQL_TEST_MYSQL_PORT", cls.config.get('database', 'port', fallback="3306"))) mysql_db = os.getenv( "PAMMYSQL_TEST_MYSQL_DB", cls.config.get('database', 'database', fallback='auth_test')) cls.dbs = pymysql.connect(host=mysql_host, user=mysql_user, password=mysql_pass, db=mysql_db, port=mysql_port) with open( os.path.join(os.path.dirname(os.path.realpath(__file__)), "testdb.sql"), "r") as f: cls.sql = f.read()
def __init__(self,lcfname): ''' Constructor ''' super(LayerFileReader,self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename)
def get(self, section, option, default=None): """ A get method which returns the default argument when the option cannot be found instead of raising an exception. """ try: value = ConfigParser.get(self, section, option) except (NoOptionError, NoSectionError): # pragma: no cover value = default return value
def loadTemplates(injectSection = None): global TEMPLATES TEMPLATES = {} parser = ConfigParser() parser.read('templates.conf') for sectionName in parser.sections(): TEMPLATES[sectionName] = Configlet(sectionName, dict(parser.items(sectionName)), injectSection)
def get_value_from_ini_config(overcloud_node, config_path, check_section, check_value, multi_key_values=False): """Get value from INI configuration file :param overcloud_node: The node that config should be pulled from :param config_path: The path of the configuration file :param check_section: Section within the config :param check_value: Value that should be checked within the config The variable could hold multiple values separated by comma. :param multi_key_values: Flag on request to hold multiple values for single key from ini file :return return_value """ class M(OrderedDict): def __setitem__(self, key, value): v_val = self.get(key) if v_val is not None and type(value) == list: v_val.append(value[0]) else: v_val = value # still using python2.7 super, for backport portability super(M, self).__setitem__(key, v_val) ini_config = get_overcloud_config(overcloud_node, config_path) config_parser_args = {'allow_no_value': True} if multi_key_values: config_parser_args['dict_type'] = M config_parser_args['strict'] = False get_value = ConfigParser(**config_parser_args) get_value.read_file(StringIO(ini_config)) value_data = [] for value in check_value.split(','): value_data.append(get_value.get(check_section, value)) return ','.join(value_data)
class ExcaliburConfigParser(ConfigParser): def __init__(self, default_config=None, *args, **kwargs): super(ExcaliburConfigParser, self).__init__(*args, **kwargs) self.excalibur_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.excalibur_defaults.read_string(default_config) self.is_validated = False def _validate(self): if self.get( "core", "executor") != "SequentialExecutor" and "sqlite" in self.get( "core", "sql_alchemy_conn"): raise ValueError("Cannot use sqlite with the {}".format( self.get("core", "executor"))) self.is_validated = True def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() if super(ExcaliburConfigParser, self).has_option(section, key): return expand_env_var( super(ExcaliburConfigParser, self).get(section, key, **kwargs)) if self.excalibur_defaults.has_option(section, key): return expand_env_var( self.excalibur_defaults.get(section, key, **kwargs)) else: raise ValueError("section/key [{section}/{key}] not found in" " config".format(**locals())) def read(self, filename): super(ExcaliburConfigParser, self).read(filename) self._validate()
def __init__(self): ''' Constructor ''' self.dvalue = None self.dselect = 'dest' #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02 self.plist = ('lgvalue','uconf','epsg','fd','td') self.cp = ConfigParser() self.fn = os.path.join(os.path.dirname(__file__),self.GUI_PREFS) with codecs.open(self.fn,'r','utf-8') as cf: self.cp.readfp(cf)
def _read_config(self): """Reads the config file with the API keys which should be placed where aliexpress_api.py is""" import os file_path = os.path.join(os.path.dirname(__file__), './config.ini') config = ConfigParser() config.read(file_path) self._appkey = config.get('AliExpress', 'appkey') self._tracking_id = config.get('AliExpress', 'trackingId')
def main(): # Import api key parser = ConfigParser() parser.read('config.ini') bot_token = parser.get('Telegram', 'token') # Create the Updater and pass it your bot's token. updater = Updater(bot_token) # todo: build a menu and conversation handler updater.dispatcher.add_handler(CommandHandler('promo', promo)) updater.dispatcher.add_handler( CommandHandler('search', search, pass_args=True)) updater.dispatcher.add_handler(CommandHandler('link', link, pass_args=True)) updater.dispatcher.add_error_handler(error) # Start the Bot updater.start_polling() # Run the bot until the user presses Ctrl-C or the process receives SIGINT, # SIGTERM or SIGABRT updater.idle()
def read_from_config_stream(config_fp, source="<stream>"): """ Read config from config file (.ini, .cfg) """ from backports.configparser import ConfigParser parser = ConfigParser() parser._read(config_fp, source) source = "config[{file_name}]".format(file_name=os.path.basename(str(source))) new_config = _ConfigStore() for section in parser.sections(): for option in parser.options(section): value = parser.get(section, option) new_config.set_config_value( section, option, ConfigValue(value, source, require_parse=True) ) return new_config
def config(filename='database.ini', section='postgresql'): # create a parser parser = ConfigParser() # read config file parser.read(filename) # get section, default to postgresql db = {} if parser.has_section(section): params = parser.items(section) for param in params: db[param[0]] = param[1] else: raise Exception('Section {0} not found in the {1} file'.format( section, filename)) return db
def new_plugin_wizard(directory=None): """ Start the wizard to create a new plugin in the current working directory. """ if directory is None: print("This wizard will create a new plugin for you in the current directory.") directory = os.getcwd() else: print("This wizard will create a new plugin for you in '%s'." % directory) if os.path.exists(directory) and not os.path.isdir(directory): print("Error: The path '%s' exists but it isn't a directory" % directory) sys.exit(1) name = ask( "What should the name of your new plugin be?", validation_regex=r'^[a-zA-Z][a-zA-Z0-9 _-]*$' ).strip() module_name = name.lower().replace(' ', '_') directory_name = name.lower().replace(' ', '-') class_name = "".join([s.capitalize() for s in name.lower().split(' ')]) description = ask( "What may I use as a short (one-line) description of your plugin?" ) if PY2: default_python_version = "2+" else: default_python_version = "3" python_version = ask( "Which python version will your plugin work with? 2, 2+ or 3? I will default to " "{version} if you leave this blank.".format(version=default_python_version), valid_responses=['2', '2+', '3', ''] ) if python_version.strip() == "": python_version = default_python_version errbot_min_version = ask( "Which minimum version of errbot will your plugin work with? " "Leave blank to support any version or input CURRENT to select the " "current version (%s)" % VERSION ).strip() if errbot_min_version.upper() == "CURRENT": errbot_min_version = VERSION errbot_max_version = ask( "Which maximum version of errbot will your plugin work with? " "Leave blank to support any version or input CURRENT to select the " "current version (%s)" % VERSION ).strip() if errbot_max_version.upper() == "CURRENT": errbot_max_version = VERSION plug = ConfigParser() plug["Core"] = { "Name": name, "Module": module_name, } plug["Documentation"] = { "Description": description, } plug["Python"] = { "Version": python_version, } plug["Errbot"] = {} if errbot_min_version != "": plug["Errbot"]["Min"] = errbot_min_version if errbot_max_version != "": plug["Errbot"]["Max"] = errbot_max_version plugin_path = directory plugfile_path = os.path.join(plugin_path, module_name+".plug") pyfile_path = os.path.join(plugin_path, module_name+".py") try: if PY3 or (PY2 and not os.path.isdir(plugin_path)): os.makedirs(plugin_path, mode=0o700) except IOError as e: if e.errno != errno.EEXIST: raise if os.path.exists(plugfile_path) or os.path.exists(pyfile_path): ask( "Warning: A plugin with this name was already found at {path}\n" "If you continue, these will be overwritten.\n" "Press Ctrl+C to abort now or type in 'overwrite' to confirm overwriting of these files." "".format( path=os.path.join(directory, module_name+".{py,plug}") ), valid_responses=["overwrite"], ) with open(plugfile_path, 'w') as f: plug.write(f) with open(pyfile_path, 'w') as f: f.write(render_plugin(locals())) print("Success! You'll find your new plugin at '%s'" % plugfile_path) print("(Don't forget to include a LICENSE file if you are going to publish your plugin)")
def _readConfigFile(self,fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed self.cp = ConfigParser() with codecs.open(fn,'r','utf-8') as cf: self.cp.readfp(cf)
class Config(object): """ Base config class definuje zakladni metody pro praci s konfiguracnim souborem. """ def __init__(self, config_name="flexipy/flexipy.conf"): self.conf = ConfigParser() #use resource management api to find flexipy.conf, see docs filename = resource_filename(Requirement.parse("flexipy"), config_name) # Open the file with the correct encoding try: with codecs.open(filename, 'r', encoding='utf-8') as f: self.conf.readfp(f) except IOError: raise ValueError('Konfiguracni soubor ' + config_name + ' neexistuje nebo jste uvedli spatnou cestu.') def get_section_list(self, section_name): """ Tato privatni metoda spracuje vsechny sekce v config filu na zaklade jmena sekce a vrati list obsahujici vsechny polozky. """ result_list = [] try: section_content = self.conf.items(section_name) for key, val in section_content: result_list.append(val) except NoSectionError: raise ValueError("Config file neobsahuje sekci " + section_name) return result_list def get_server_config(self): """ Tato metoda vrati dict obsahujici vsechna nastaveni tykajici se serveru. """ result = {} try: section_content = self.conf.items("server") for key, val in section_content: result[key] = val except NoSectionError: raise ValueError("Config file neobsahuje sekci server") return result def get_evidence_list(self): return self.get_section_list('evidence') def get_typy_faktury_prijate(self): return self.get_section_list('typ_faktury_prijate') def get_typy_faktury_vydane(self): return self.get_section_list('typ_faktury_vydane') def get_typ_bank_dokladu(self): return self.get_section_list('typ_bank_dokladu') def get_typ_pohybu(self): return self.get_section_list('typ_pohybu') def get_bankovni_ucty(self): return self.get_section_list('bankovni_ucty') def get_typ_polozky_vydane(self): return self.get_section_list('typ_polozky_vydane') def get_typ_ucetni_operace(self): return self.get_section_list('typ_ucetni_operace') def get_typ_pokladni_pohyb(self): return self.get_section_list('typ_pokladni_pohyb') def get_typ_pokladna(self): return self.get_section_list('typ_pokladna')
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. # These configs can also be fetched from Secrets backend # following the "{section}__{name}__secret" pattern sensitive_config_values = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'flower_basic_auth'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', }, 'elasticsearch': { 'host': 'elasticsearch_host', 'log_id_template': 'elasticsearch_log_id_template', 'end_of_log_mark': 'elasticsearch_end_of_log_mark', 'frontend': 'elasticsearch_frontend', 'write_stdout': 'elasticsearch_write_stdout', 'json_format': 'elasticsearch_json_format', 'json_fields': 'elasticsearch_json_fields' } } # A mapping of old default values that we want to change and warn the user # about. Mapping of section -> setting -> { old, replace, by_version } deprecated_values = { 'core': { 'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'), }, } # This method transforms option names on every read, get, or set operation. # This changes from the default behaviour of ConfigParser from lowercasing # to instead be case-preserving def optionxform(self, optionstr): return optionstr def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): self._validate_config_dependencies() for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): old, new, version = info if self.get(section, name, fallback=None) == old: # Make sure the env var option is removed, otherwise it # would be read and used instead of the value we set env_var = self._env_var_name(section, name) os.environ.pop(env_var, None) self.set(section, name, new) warnings.warn( 'The {name} setting in [{section}] has the old default value ' 'of {old!r}. This value has been changed to {new!r} in the ' 'running config, but please update your config before Apache ' 'Airflow {version}.'.format( name=name, section=section, old=old, new=new, version=version ), FutureWarning ) self.is_validated = True def _validate_config_dependencies(self): """ Validate that config values aren't invalid given other config values or system-level limitations and requirements. """ if ( self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") if self.has_option('core', 'mp_start_method'): mp_start_method = self.get('core', 'mp_start_method') start_method_options = multiprocessing.get_all_start_methods() if mp_start_method not in start_method_options: raise AirflowConfigException( "mp_start_method should not be " + mp_start_method + ". Possible values are " + ", ".join(start_method_options)) @staticmethod def _env_var_name(section, key): return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) # alternatively AIRFLOW__{SECTION}__{KEY}_CMD (for a command) env_var_cmd = env_var + '_CMD' if env_var_cmd in os.environ: # if this is a valid command key... if (section, key) in self.sensitive_config_values: return run_command(os.environ[env_var_cmd]) # alternatively AIRFLOW__{SECTION}__{KEY}_SECRET (to get from Secrets Backend) env_var_secret_path = env_var + '_SECRET' if env_var_secret_path in os.environ: # if this is a valid secret path... if (section, key) in self.sensitive_config_values: return _get_config_value_from_secret_backend(os.environ[env_var_secret_path]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.sensitive_config_values: if super(AirflowConfigParser, self).has_option(section, fallback_key): command = super(AirflowConfigParser, self).get(section, fallback_key) return run_command(command) def _get_secret_option(self, section, key): """Get Config option values from Secret Backend""" fallback_key = key + '_secret' # if this is a valid secret key... if (section, key) in self.sensitive_config_values: if super(AirflowConfigParser, self).has_option(section, fallback_key): secrets_path = super(AirflowConfigParser, self).get(section, fallback_key) return _get_config_value_from_secret_backend(secrets_path) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var(super(AirflowConfigParser, self).get( section, deprecated_name, **kwargs )) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then from secret backends option = self._get_secret_option(section, key) if option: return option if deprecated_name: option = self._get_secret_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning( "section/key [%s/%s] not found in config", section, key ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(section=section, key=key)) def getimport(self, section, key, **kwargs): """ Reads options, imports the full qualified name, and returns the object. In case of failure, it throws an exception a clear message with the key aad the section names :return: The object or None, if the option is empty """ full_qualified_path = conf.get(section=section, key=key, **kwargs) if not full_qualified_path: return None try: return import_string(full_qualified_path) except ImportError as e: log.error(e) raise AirflowConfigException( 'The object could not be loaded. Please check "{key}" key in "{section}" section. ' 'Current value: "{full_qualified_path}".'.format( key=key, section=section, full_qualified_path=full_qualified_path) ) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: raise ValueError( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super(AirflowConfigParser, self).read(filenames, **kwargs) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except (NoOptionError, NoSectionError): return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '') if key.endswith("_CMD"): key = key[:-4] key = key.lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def write(self, fp, space_around_delimiters=True): # This is based on the configparser.RawConfigParser.write method code to add support for # reading options from environment variables. if space_around_delimiters: d = " {} ".format(self._delimiters[0]) # type: ignore else: d = self._delimiters[0] # type: ignore if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) # type: ignore for section in self._sections: self._write_section(fp, section, self.getsection(section).items(), d) # type: ignore def as_dict( self, display_source=False, display_sensitive=False, raw=False, include_env=True, include_cmds=True, include_secret=True ): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool :param include_env: Should the value of configuration from AIRFLOW__ environment variables be included or not :type include_env: bool :param include_cmds: Should the result of calling any *_cmd config be set (True, default), or should the _cmd options be left as the command to run (False) :type include_cmds: bool :param include_secret: Should the result of calling any *_secret config be set (True, default), or should the _secret options be left as the path to get the secret from (False) :type include_secret: bool :return: Dictionary, where the key is the name of the section and the content is the dictionary with the name of the parameter and its value. """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority if include_env: for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__', 2) opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') section = section.lower() # if we lower key for kubernetes_environment_variables section, # then we won't be able to set any Airflow environment # variables. Airflow only parse environment variables starts # with AIRFLOW_. Therefore, we need to make it a special case. if section != 'kubernetes_environment_variables': key = key.lower() cfg.setdefault(section, OrderedDict()).update({key: opt}) # add bash commands if include_cmds: for (section, key) in self.sensitive_config_values: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] # add config from secret backends if include_secret: for (section, key) in self.sensitive_config_values: opt = self._get_secret_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'secret') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_secret'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults log.info("Overriding settings with defaults from %s", DEFAULT_CONFIG_FILE_PATH) self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config log.info("Reading default test configuration from %s", TEST_CONFIG_FILE_PATH) self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings log.info("Reading test configuration from %s", TEST_CONFIG_FILE) self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.'.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
def new_plugin_wizard(directory=None): """ Start the wizard to create a new plugin in the current working directory. """ if directory is None: print( "This wizard will create a new plugin for you in the current directory." ) directory = os.getcwd() else: print("This wizard will create a new plugin for you in '%s'." % directory) if os.path.exists(directory) and not os.path.isdir(directory): print("Error: The path '%s' exists but it isn't a directory" % directory) sys.exit(1) name = ask("What should the name of your new plugin be?", validation_regex=r'^[a-zA-Z][a-zA-Z0-9 _-]*$').strip() module_name = name.lower().replace(' ', '_') directory_name = name.lower().replace(' ', '-') class_name = "".join([s.capitalize() for s in name.lower().split(' ')]) description = ask( "What may I use as a short (one-line) description of your plugin?") if PY2: default_python_version = "2+" else: default_python_version = "3" python_version = ask( "Which python version will your plugin work with? 2, 2+ or 3? I will default to " "{version} if you leave this blank.".format( version=default_python_version), valid_responses=['2', '2+', '3', '']) if python_version.strip() == "": python_version = default_python_version errbot_min_version = ask( "Which minimum version of errbot will your plugin work with? " "Leave blank to support any version or input CURRENT to select the " "current version (%s)" % VERSION).strip() if errbot_min_version.upper() == "CURRENT": errbot_min_version = VERSION errbot_max_version = ask( "Which maximum version of errbot will your plugin work with? " "Leave blank to support any version or input CURRENT to select the " "current version (%s)" % VERSION).strip() if errbot_max_version.upper() == "CURRENT": errbot_max_version = VERSION plug = ConfigParser() plug["Core"] = { "Name": name, "Module": module_name, } plug["Documentation"] = { "Description": description, } plug["Python"] = { "Version": python_version, } plug["Errbot"] = {} if errbot_min_version != "": plug["Errbot"]["Min"] = errbot_min_version if errbot_max_version != "": plug["Errbot"]["Max"] = errbot_max_version plugin_path = directory plugfile_path = os.path.join(plugin_path, module_name + ".plug") pyfile_path = os.path.join(plugin_path, module_name + ".py") try: if PY3 or (PY2 and not os.path.isdir(plugin_path)): os.makedirs(plugin_path, mode=0o700) except IOError as e: if e.errno != errno.EEXIST: raise if os.path.exists(plugfile_path) or os.path.exists(pyfile_path): ask( "Warning: A plugin with this name was already found at {path}\n" "If you continue, these will be overwritten.\n" "Press Ctrl+C to abort now or type in 'overwrite' to confirm overwriting of these files." "".format(path=os.path.join(directory, module_name + ".{py,plug}")), valid_responses=["overwrite"], ) with open(plugfile_path, 'w') as f: plug.write(f) with open(pyfile_path, 'w') as f: f.write(render_plugin(locals())) print("Success! You'll find your new plugin at '%s'" % plugfile_path) print( "(Don't forget to include a LICENSE file if you are going to publish your plugin)" )
class GUIPrefsReader(object): ''' Reader for GUI prefs. To save re inputting every time ''' PREFS_SEC = 'prefs' GUI_PREFS = '../conf/gui.prefs' def __init__(self): ''' Constructor ''' self.dvalue = None self.dselect = 'dest' #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02 self.plist = ('lgvalue', 'uconf', 'epsg', 'fd', 'td') self.cp = ConfigParser() self.fn = os.path.join(os.path.dirname(__file__), self.GUI_PREFS) with codecs.open(self.fn, 'r', 'utf-8') as cf: self.cp.readfp(cf) def read(self): '''Read stored DS value and return this and its matching params''' try: with codecs.open(self.fn, 'r', 'utf-8') as cf: self.cp.readfp(cf) self.dvalue = self.cp.get(self.PREFS_SEC, self.dselect) if LU.assessNone(self.dvalue) is None: return (None, ) * (len(self.plist) + 1) except NoSectionError as nse: #if no sec init sec and opt and ret nones ldslog.warn('Error getting GUI prefs section :: ' + str(nse)) if not self._initSection(self.PREFS_SEC): raise self._initOption(self.PREFS_SEC, self.dselect) return (None, ) * (len(self.plist) + 1) except NoOptionError as noe: #if no opt init opt and ret nones ldslog.warn('Error getting GUI prefs :: ' + str(noe)) if not self._initOption(self.PREFS_SEC, self.dselect): raise return (None, ) * (len(self.plist) + 1) #if dval is okay ret it and res of a read of that sec return (self.dvalue, ) + self.readsec(self.dvalue) def readall(self): '''Reads entire grp into dict''' gpra = {} secs = self.cp.sections() secs.remove(self.PREFS_SEC) for sec in secs: gpra[sec] = self.readsec(sec) return gpra def getDestinations(self): return self.cp.sections() def readsec(self, section): #options per DS type rlist = () for p in self.plist: try: rlist += (self.cp.get(section, p), ) except NoSectionError as nse: #if not ds sec init sec then init opt ldslog.warn('Error getting GUI ' + section + ' :: ' + str(nse)) if not self._initSection(section): raise self._initOption(section, p) rlist += (None, ) except NoOptionError as noe: #if no opt init the opt ldslog.warn('Error getting GUI ' + section + ' pref, ' + p + ' :: ' + str(noe)) if not self._initOption(section, p): raise rlist += (None, ) return rlist def writeline(self, field, value): #not the best solution since depends on a current gpr and a recent read/write. if self.dvalue: self.writesecline(self.dvalue, field, value) def writesecline(self, section, field, value): try: self.cp.set(section, field, value if LU.assessNone(value) else '') with codecs.open(self.fn, 'w', 'utf-8') as configfile: self.cp.write(configfile) ldslog.debug(str(section) + ':' + str(field) + '=' + str(value)) except Exception as e: ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format( e, (section, field, value))) def write(self, rlist): self.dvalue = rlist[0] if self.cp.has_section(self.PREFS_SEC): self.cp.set(self.PREFS_SEC, self.dselect, self.dvalue) else: self.cp.add_section(self.PREFS_SEC) self.cp.set(self.PREFS_SEC, self.dselect, self.dvalue) for pr in zip(self.plist, rlist[1:]): if not self.cp.has_section(self.dvalue): self.cp.add_section(self.dvalue) try: if LU.assessNone(pr[1]): self.cp.set(self.dvalue, pr[0], pr[1]) ldslog.debug(self.dvalue + ':' + pr[0] + '=' + pr[1]) except Exception as e: ldslog.warn('Problem writing GUI prefs. ' + str(e)) with codecs.open(self.fn, 'w', 'utf-8') as configfile: self.cp.write(configfile) def _initSection(self, section): checksec = LU.standardiseDriverNames(section) if checksec: self.cp.add_section(checksec) return True elif section == self.PREFS_SEC: self.cp.add_section(section) return True return False def _initOption(self, section, option): if option in self.plist + (self.dselect, ): self.writesecline(section, option, None) return True return False @classmethod def validate(): '''Make sure a guipref file is valid, check pref points to alt least one valid DST''' filename = os.path.join(os.path.dirname(__file__), GUIPrefsReader.GUI_PREFS) gp = GUIPrefsReader(filename) #validate UC check it has a valid dest named and configured p = gp.cp.get('prefs', 'dest') return gp.cp.has_section(p)
class SpacetimeGISConfigParser(ConfigParser): def __init__(self, default_config=None, *args, **kwargs): super().__init__(*args, **kwargs) self.spacetimegis_defaults = ConfigParser(*args, **kwargs) if not default_config: self.spacetimegis_defaults.read_string(default_config) @staticmethod def _env_var_name(section, key): return 'SPACETIMEGIS__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format SPACETIMEGIS__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super().has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var(super().get(section, key, **kwargs)) # ...then the default config if self.spacetimegis_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.spacetimegis_defaults.get(section, key, **kwargs)) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: # logger.writelog(LogLevel.error, # 'The value for configuration option "{}:{}" is not a ' # 'boolean (received "{}").'.format(section, key, val)) raise ValueError( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super().read(filenames, **kwargs) def read_dict(self, *args, **kwargs): super().read_dict(*args, **kwargs) def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except NoOptionError: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super().has_option(section, option): super().remove_option(section, option) if self.spacetimegis_defaults.has_option(section, option) and remove_default: self.spacetimegis_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.spacetimegis_defaults._sections): return None _section = copy.deepcopy(self.spacetimegis_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'SPACETIMEGIS__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.spacetimegis_defaults), ('spacetimegis.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k.upper()] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('SPACETIMEGIS__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'SPACETIMEGIS__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) return cfg def as_all_dict(self, display_source=False, display_sensitive=False, raw=False): tmp = self.as_dict(display_source, display_sensitive, raw) all_cfg = {} for val in tmp.values(): if not all_cfg: all_cfg = val.copy() else: all_cfg.update(val) return all_cfg
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = {('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend')} def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False def _validate(self): if (self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif (self.getboolean("webserver", "authenticate") and self.get( "webserver", "owner_mode") not in ['user', 'ldapgroup']): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif (self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ('airflow.contrib.auth.backends.ldap_auth')): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option # ...then the default config if self.defaults.has_option(section, key): return expand_env_var(self.defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format( **locals())) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.defaults.has_option(section, option) and remove_default: self.defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if section not in self._sections and section not in self.defaults._sections: return None _section = copy.deepcopy(self.defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg' or 'default'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool """ cfg = copy.deepcopy(self.defaults._sections) cfg.update(copy.deepcopy(self._sections)) # remove __name__ (affects Python 2 only) for options in cfg.values(): options.pop('__name__', None) # add source if display_source: for section in cfg: for k, v in cfg[section].items(): cfg[section][k] = (v, 'airflow config') # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: opt = None if opt: if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'bash cmd') cfg.setdefault(section, OrderedDict()).update({key: opt}) return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.defaults.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE)
class GUIPrefsReader(object): ''' Reader for GUI prefs. To save re inputting every time ''' PREFS_SEC = 'prefs' GUI_PREFS = '../conf/gui.prefs' def __init__(self): ''' Constructor ''' self.dvalue = None self.dselect = 'dest' #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02 self.plist = ('lgvalue','uconf','epsg','fd','td') self.cp = ConfigParser() self.fn = os.path.join(os.path.dirname(__file__),self.GUI_PREFS) with codecs.open(self.fn,'r','utf-8') as cf: self.cp.readfp(cf) def read(self): '''Read stored DS value and return this and its matching params''' try: with codecs.open(self.fn, 'r','utf-8') as cf: self.cp.readfp(cf) self.dvalue = self.cp.get(self.PREFS_SEC, self.dselect) if LU.assessNone(self.dvalue) is None: return (None,)*(len(self.plist)+1) except NoSectionError as nse: #if no sec init sec and opt and ret nones ldslog.warn('Error getting GUI prefs section :: '+str(nse)) if not self._initSection(self.PREFS_SEC): raise self._initOption(self.PREFS_SEC,self.dselect) return (None,)*(len(self.plist)+1) except NoOptionError as noe: #if no opt init opt and ret nones ldslog.warn('Error getting GUI prefs :: '+str(noe)) if not self._initOption(self.PREFS_SEC,self.dselect): raise return (None,)*(len(self.plist)+1) #if dval is okay ret it and res of a read of that sec return (self.dvalue,)+self.readsec(self.dvalue) def readall(self): '''Reads entire grp into dict''' gpra = {} secs = self.cp.sections() secs.remove(self.PREFS_SEC) for sec in secs: gpra[sec] = self.readsec(sec) return gpra def getDestinations(self): return self.cp.sections() def readsec(self,section): #options per DS type rlist = () for p in self.plist: try: rlist += (self.cp.get(section, p),) except NoSectionError as nse: #if not ds sec init sec then init opt ldslog.warn('Error getting GUI '+section+' :: '+str(nse)) if not self._initSection(section): raise self._initOption(section, p) rlist += (None,) except NoOptionError as noe: #if no opt init the opt ldslog.warn('Error getting GUI '+section+' pref, '+p+' :: '+str(noe)) if not self._initOption(section,p): raise rlist += (None,) return rlist def writeline(self,field,value): #not the best solution since depends on a current gpr and a recent read/write. if self.dvalue: self.writesecline(self.dvalue,field,value) def writesecline(self,section,field,value): try: self.cp.set(section,field,value if LU.assessNone(value) else '') with codecs.open(self.fn, 'w','utf-8') as configfile: self.cp.write(configfile) ldslog.debug(str(section)+':'+str(field)+'='+str(value)) except Exception as e: ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format(e,(section,field,value))) def write(self,rlist): self.dvalue = rlist[0] if self.cp.has_section(self.PREFS_SEC): self.cp.set(self.PREFS_SEC,self.dselect,self.dvalue) else: self.cp.add_section(self.PREFS_SEC) self.cp.set(self.PREFS_SEC,self.dselect,self.dvalue) for pr in zip(self.plist,rlist[1:]): if not self.cp.has_section(self.dvalue): self.cp.add_section(self.dvalue) try: if LU.assessNone(pr[1]): self.cp.set(self.dvalue,pr[0],pr[1]) ldslog.debug(self.dvalue+':'+pr[0]+'='+pr[1]) except Exception as e: ldslog.warn('Problem writing GUI prefs. '+str(e)) with codecs.open(self.fn, 'w','utf-8') as configfile: self.cp.write(configfile) def _initSection(self,section): checksec = LU.standardiseDriverNames(section) if checksec: self.cp.add_section(checksec) return True elif section == self.PREFS_SEC: self.cp.add_section(section) return True return False def _initOption(self,section,option): if option in self.plist+(self.dselect,): self.writesecline(section,option,None) return True return False @classmethod def validate(): '''Make sure a guipref file is valid, check pref points to alt least one valid DST''' filename = os.path.join(os.path.dirname(__file__),GUIPrefsReader.GUI_PREFS) gp = GUIPrefsReader(filename) #validate UC check it has a valid dest named and configured p = gp.cp.get('prefs', 'dest') return gp.cp.has_section(p)
class MainFileReader(object): ''' Config file reader/writer ''' LDSN = 'LDS' PROXY = 'Proxy' MISC = 'Misc' DEFAULT_MF = 'template.conf' def __init__(self,cfpath=None,use_defaults=True): ''' Constructor ''' from lds.DataStore import DataStore self.driverconfig = {i:() for i in DataStore.DRIVER_NAMES} self.use_defaults = use_defaults #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template if cfpath is None: self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF) else: #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry) self.filename = LU.standardiseUserConfigName(cfpath) self.cp = None self.initMainFile() self.fn = re.search('(.+)\.conf',os.path.basename(self.filename)).group(1) def __str__(self): return self.filename def initMainFile(self,template=''): '''Open and populate a new config file with 'template' else just touch it. Then call the reader''' if self.filename.split('/')[-1] != self.DEFAULT_MF: mode = 'a' if template is '' else 'w' with codecs.open(self.filename, mode, 'utf-8') as f: f.write(template) self._readConfigFile(self.filename) def hasSection(self,secname): return secname in self.getSections() def getSections(self): '''List of sections (layernames/datasources)''' return self.cp.sections() def _readConfigFile(self,fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed self.cp = ConfigParser() with codecs.open(fn,'r','utf-8') as cf: self.cp.readfp(cf) #database def readDriverConfig(self,dname): dnd = {'PostgreSQL':self.readPostgreSQLConfig, 'MSSQLSpatial':self.readMSSQLConfig, 'FileGDB':self.readFileGDBConfig, 'SQLite':self.readSpatiaLiteConfig, 'WFS':self.readWFSConfig, 'Proxy':self.readProxyConfig, 'Misc':self.readMiscConfig} return dnd[dname]() def readPostgreSQLConfig(self): '''PostgreSQL specific config file reader''' from PostgreSQLDataStore import PostgreSQLDataStore as PG ref = self.fn+':PG. ' usr = None pwd = None over = None epsg = None cql = None schema = None if self.use_defaults: host = "127.0.0.1" port = 5432 dbname = "ldsincr" schema = "lds" config = "external" else: host = None port = None dbname = None config = None try: host = self.cp.get(PG.DRIVER_NAME, 'host') except NoSectionError: ldslog.warn(ref+"No PostgreSQL section") raise #return (None,)*10 except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: port = self.cp.get(PG.DRIVER_NAME, 'port') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: dbname = self.cp.get(PG.DRIVER_NAME, 'dbname') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: schema = self.cp.get(PG.DRIVER_NAME, 'schema') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: usr = self.cp.get(PG.DRIVER_NAME, 'user') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: pwd = self.cp.get(PG.DRIVER_NAME, 'pass') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: config = self.cp.get(PG.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref+"No config preference specified, default to "+str(config)) try: over = self.cp.get(PG.DRIVER_NAME, 'overwrite') except NoOptionError: ldslog.warn(ref+"Overwrite not specified, Setting to True") over = True try: epsg = self.cp.get(PG.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref+"EPSG not specified, default to none keeping existing SRS") epsg = True try: cql = self.cp.get(PG.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref+"No CQL Filter specified, fetching all results") cql = None return (host,port,dbname,schema,usr,pwd,over,config,epsg,cql) def readMSSQLConfig(self): '''MSSQL specific config file reader''' from MSSQLSpatialDataStore import MSSQLSpatialDataStore as MS ref = self.fn+':MS. ' usr = None pwd = None epsg = None cql = None schema = None if self.use_defaults: odbc = "FreeTDS" server = "127.0.0.1\SQLExpress" dbname = "LDSINCR" trust = "yes" dsn = "LDSINCR" config = "external" else: odbc = None server = None dbname = None trust = None dsn = None config = None try: odbc = self.cp.get(MS.DRIVER_NAME, 'odbc') except NoSectionError: ldslog.warn(ref+"No MSSQLSpatial section") raise #return (None,)*11 except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: server = self.cp.get(MS.DRIVER_NAME, 'server') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: dsn = self.cp.get(MS.DRIVER_NAME, 'dsn') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: trust = self.cp.get(MS.DRIVER_NAME, 'trust') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: dbname = self.cp.get(MS.DRIVER_NAME, 'dbname') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: schema = self.cp.get(MS.DRIVER_NAME, 'schema') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: usr = self.cp.get(MS.DRIVER_NAME, 'user') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: pwd = self.cp.get(MS.DRIVER_NAME, 'pass') except NoOptionError as noe: ldslog.warn(ref+str(noe)) try: config = self.cp.get(MS.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref+"No config preference specified, default to "+str(config)) try: epsg = self.cp.get(MS.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref+"EPSG not specified, default to None keeping existing SRS") try: cql = self.cp.get(MS.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref+"No CQL Filter specified, fetching all results") cql = None return (odbc,server,dsn,trust,dbname,schema,usr,pwd,config,epsg,cql) def readSpatiaLiteConfig(self): '''SpatiaLite specific config file reader''' from SpatiaLiteDataStore import SpatiaLiteDataStore as SL ref = self.fn+':SL. ' epsg = None cql = None if self.use_defaults: lcfname = "~/LDSSLITE.sqlite3" config = "external" else: lcfname = None config = None try: lcfname = self.cp.get(SL.DRIVER_NAME, 'file') except NoSectionError: ldslog.warn(ref+"No SpatiaLite section") raise #return (None,)*4 except NoOptionError: ldslog.warn(ref+"No DB name provided, default to "+str(lcfname)) try: config = self.cp.get(SL.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref+"No config preference specified, default to "+str(config)) try: epsg = self.cp.get(SL.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref+"EPSG not specified, default to "+str(epsg)+" keeping existing SRS") try: cql = self.cp.get(SL.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref+"No CQL Filter specified, fetching all results") return (lcfname,config,epsg,cql) def readFileGDBConfig(self): '''FileGDB specific config file reader''' from FileGDBDataStore import FileGDBDataStore as FG ref = self.fn+':FG. ' epsg = None cql = None if self.use_defaults: lcfname = "~/LDSFGDB.gdb" config = "external" else: lcfname = None config = None try: lcfname = self.cp.get(FG.DRIVER_NAME, 'file') except NoSectionError: ldslog.warn(ref+"No FileGDB section") raise #return (None,)*4 except NoOptionError: ldslog.warn(ref+"No DB name provided, default to "+str(lcfname)) try: config = self.cp.get(FG.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref+"No config preference specified, default to "+str(config)) try: epsg = self.cp.get(FG.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref+"EPSG not specified, default to "+str(epsg)+" none keeping existing SRS") try: cql = self.cp.get(FG.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref+"No CQL Filter specified, fetching all results") return (lcfname,config,epsg,cql) def readWFSConfig(self): '''Generic WFS config file reader''' '''Since this now keys on the driver name, WFS is read before LDS and LDS not at all, So...''' return self.readLDSConfig() def readLDSConfig(self): '''LDS specific config file reader''' #***HACK *** reimport NSE aliased from ConfigParser import NoSectionError as NSE #use_defaults determines whether we use default values. For a user config this may not be wise #since a user config is a custom file relying on the main config for absent values not last-resort defaults ref = self.fn+':LDS. ' cql = None key = None #url = "http://data.linz.govt.nz/" #ver = "2.0.0" if self.use_defaults: url = "http://wfs.data.linz.govt.nz/" fmt = "GML2" svc = "WFS" ver = "1.0.0" else: url = None fmt = None svc = None ver = None try: url = self.cp.get(self.LDSN, 'url') #for some reason we cant use global NoSectionError in this function (even though its fine everywhere else) except NSE: ldslog.warn(ref+"LDS: No LDS Section") return (None,)*6 except NoOptionError: ldslog.warn(ref+"Default URL assumed ") try: key = self.cp.get(self.LDSN, 'key') except NoOptionError, NoSectionError: ldslog.error(ref+"Key required to connect to LDS...") raise try: fmt = self.cp.get(self.LDSN, 'fmt') except NoOptionError: ldslog.warn(ref+"No output format specified") try: svc = self.cp.get(self.LDSN, 'svc') except NoOptionError: ldslog.warn(ref+"No service type specified, default to "+str(svc)) try: ver = self.cp.get(self.LDSN, 'ver') except NoOptionError: ldslog.warn(ref+"No Version specified, assuming WFS and default to version "+str(ver)) try: cql = self.cp.get(self.LDSN, 'cql') except NoOptionError: ldslog.warn(ref+"No CQL Filter specified, fetching all results") return (url,key,svc,ver,fmt,cql)
class MainFileReader(object): ''' Config file reader/writer ''' LDSN = 'LDS' PROXY = 'Proxy' MISC = 'Misc' DEFAULT_MF = 'template.conf' def __init__(self, cfpath=None, use_defaults=True): ''' Constructor ''' from lds.DataStore import DataStore self.driverconfig = {i: () for i in DataStore.DRIVER_NAMES} self.use_defaults = use_defaults #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template if cfpath is None: self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF) else: #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry) self.filename = LU.standardiseUserConfigName(cfpath) self.cp = None self.initMainFile() self.fn = re.search('(.+)\.conf', os.path.basename(self.filename)).group(1) def __str__(self): return self.filename def initMainFile(self, template=''): '''Open and populate a new config file with 'template' else just touch it. Then call the reader''' if self.filename.split('/')[-1] != self.DEFAULT_MF: mode = 'a' if template is '' else 'w' with codecs.open(self.filename, mode, 'utf-8') as f: f.write(template) self._readConfigFile(self.filename) def hasSection(self, secname): return secname in self.getSections() def getSections(self): '''List of sections (layernames/datasources)''' return self.cp.sections() def _readConfigFile(self, fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed self.cp = ConfigParser() with codecs.open(fn, 'r', 'utf-8') as cf: self.cp.readfp(cf) #database def readDriverConfig(self, dname): dnd = { 'PostgreSQL': self.readPostgreSQLConfig, 'MSSQLSpatial': self.readMSSQLConfig, 'FileGDB': self.readFileGDBConfig, 'SQLite': self.readSpatiaLiteConfig, 'WFS': self.readWFSConfig, 'Proxy': self.readProxyConfig, 'Misc': self.readMiscConfig } return dnd[dname]() def readPostgreSQLConfig(self): '''PostgreSQL specific config file reader''' from PostgreSQLDataStore import PostgreSQLDataStore as PG ref = self.fn + ':PG. ' usr = None pwd = None over = None epsg = None cql = None schema = None if self.use_defaults: host = "127.0.0.1" port = 5432 dbname = "ldsincr" schema = "lds" config = "external" else: host = None port = None dbname = None config = None try: host = self.cp.get(PG.DRIVER_NAME, 'host') except NoSectionError: ldslog.warn(ref + "No PostgreSQL section") raise #return (None,)*10 except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: port = self.cp.get(PG.DRIVER_NAME, 'port') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: dbname = self.cp.get(PG.DRIVER_NAME, 'dbname') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: schema = self.cp.get(PG.DRIVER_NAME, 'schema') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: usr = self.cp.get(PG.DRIVER_NAME, 'user') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: pwd = self.cp.get(PG.DRIVER_NAME, 'pass') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: config = self.cp.get(PG.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref + "No config preference specified, default to " + str(config)) try: over = self.cp.get(PG.DRIVER_NAME, 'overwrite') except NoOptionError: ldslog.warn(ref + "Overwrite not specified, Setting to True") over = True try: epsg = self.cp.get(PG.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn( ref + "EPSG not specified, default to none keeping existing SRS") epsg = True try: cql = self.cp.get(PG.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref + "No CQL Filter specified, fetching all results") cql = None return (host, port, dbname, schema, usr, pwd, over, config, epsg, cql) def readMSSQLConfig(self): '''MSSQL specific config file reader''' from MSSQLSpatialDataStore import MSSQLSpatialDataStore as MS ref = self.fn + ':MS. ' usr = None pwd = None epsg = None cql = None schema = None if self.use_defaults: odbc = "FreeTDS" server = "127.0.0.1\SQLExpress" dbname = "LDSINCR" trust = "yes" dsn = "LDSINCR" config = "external" else: odbc = None server = None dbname = None trust = None dsn = None config = None try: odbc = self.cp.get(MS.DRIVER_NAME, 'odbc') except NoSectionError: ldslog.warn(ref + "No MSSQLSpatial section") raise #return (None,)*11 except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: server = self.cp.get(MS.DRIVER_NAME, 'server') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: dsn = self.cp.get(MS.DRIVER_NAME, 'dsn') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: trust = self.cp.get(MS.DRIVER_NAME, 'trust') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: dbname = self.cp.get(MS.DRIVER_NAME, 'dbname') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: schema = self.cp.get(MS.DRIVER_NAME, 'schema') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: usr = self.cp.get(MS.DRIVER_NAME, 'user') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: pwd = self.cp.get(MS.DRIVER_NAME, 'pass') except NoOptionError as noe: ldslog.warn(ref + str(noe)) try: config = self.cp.get(MS.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref + "No config preference specified, default to " + str(config)) try: epsg = self.cp.get(MS.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn( ref + "EPSG not specified, default to None keeping existing SRS") try: cql = self.cp.get(MS.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref + "No CQL Filter specified, fetching all results") cql = None return (odbc, server, dsn, trust, dbname, schema, usr, pwd, config, epsg, cql) def readSpatiaLiteConfig(self): '''SpatiaLite specific config file reader''' from SpatiaLiteDataStore import SpatiaLiteDataStore as SL ref = self.fn + ':SL. ' epsg = None cql = None if self.use_defaults: lcfname = "~/LDSSLITE.sqlite3" config = "external" else: lcfname = None config = None try: lcfname = self.cp.get(SL.DRIVER_NAME, 'file') except NoSectionError: ldslog.warn(ref + "No SpatiaLite section") raise #return (None,)*4 except NoOptionError: ldslog.warn(ref + "No DB name provided, default to " + str(lcfname)) try: config = self.cp.get(SL.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref + "No config preference specified, default to " + str(config)) try: epsg = self.cp.get(SL.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref + "EPSG not specified, default to " + str(epsg) + " keeping existing SRS") try: cql = self.cp.get(SL.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref + "No CQL Filter specified, fetching all results") return (lcfname, config, epsg, cql) def readFileGDBConfig(self): '''FileGDB specific config file reader''' from FileGDBDataStore import FileGDBDataStore as FG ref = self.fn + ':FG. ' epsg = None cql = None if self.use_defaults: lcfname = "~/LDSFGDB.gdb" config = "external" else: lcfname = None config = None try: lcfname = self.cp.get(FG.DRIVER_NAME, 'file') except NoSectionError: ldslog.warn(ref + "No FileGDB section") raise #return (None,)*4 except NoOptionError: ldslog.warn(ref + "No DB name provided, default to " + str(lcfname)) try: config = self.cp.get(FG.DRIVER_NAME, 'config') except NoOptionError: ldslog.warn(ref + "No config preference specified, default to " + str(config)) try: epsg = self.cp.get(FG.DRIVER_NAME, 'epsg') except NoOptionError: ldslog.warn(ref + "EPSG not specified, default to " + str(epsg) + " none keeping existing SRS") try: cql = self.cp.get(FG.DRIVER_NAME, 'cql') except NoOptionError: ldslog.warn(ref + "No CQL Filter specified, fetching all results") return (lcfname, config, epsg, cql) def readWFSConfig(self): '''Generic WFS config file reader''' '''Since this now keys on the driver name, WFS is read before LDS and LDS not at all, So...''' return self.readLDSConfig() def readLDSConfig(self): '''LDS specific config file reader''' #***HACK *** reimport NSE aliased from ConfigParser import NoSectionError as NSE #use_defaults determines whether we use default values. For a user config this may not be wise #since a user config is a custom file relying on the main config for absent values not last-resort defaults ref = self.fn + ':LDS. ' cql = None key = None #url = "http://data.linz.govt.nz/" #ver = "2.0.0" if self.use_defaults: url = "http://wfs.data.linz.govt.nz/" fmt = "GML2" svc = "WFS" ver = "1.0.0" else: url = None fmt = None svc = None ver = None try: url = self.cp.get(self.LDSN, 'url') #for some reason we cant use global NoSectionError in this function (even though its fine everywhere else) except NSE: ldslog.warn(ref + "LDS: No LDS Section") return (None, ) * 6 except NoOptionError: ldslog.warn(ref + "Default URL assumed ") try: key = self.cp.get(self.LDSN, 'key') except NoOptionError, NoSectionError: ldslog.error(ref + "Key required to connect to LDS...") raise try: fmt = self.cp.get(self.LDSN, 'fmt') except NoOptionError: ldslog.warn(ref + "No output format specified") try: svc = self.cp.get(self.LDSN, 'svc') except NoOptionError: ldslog.warn(ref + "No service type specified, default to " + str(svc)) try: ver = self.cp.get(self.LDSN, 'ver') except NoOptionError: ldslog.warn( ref + "No Version specified, assuming WFS and default to version " + str(ver)) try: cql = self.cp.get(self.LDSN, 'cql') except NoOptionError: ldslog.warn(ref + "No CQL Filter specified, fetching all results") return (url, key, svc, ver, fmt, cql)
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', } } deprecation_format_string = ( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.' ) def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): if ( self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var(super(AirflowConfigParser, self).get( section, deprecated_name, **kwargs )) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key): return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format(**locals()) ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict( self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update( {key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( self.deprecation_format_string.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
def loadConfig(): global CONFIG CONFIG = {} CONFIG = ConfigParser() CONFIG.read('global.conf')
class LayerFileReader(LayerReader): def __init__(self, lcfname): ''' Constructor ''' super(LayerFileReader, self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename) def isCurrent(self): '''TF test to decide whether to init''' return self.lcfilename and self._fileexists() and len( self.cp.sections()) > 0 def close(self): self.cp = None self.lcfilename = None self.lcfname = None def _fileexists(self): return os.path.exists(self.lcfilename) def buildConfigLayer(self, res): '''Just write a file in conf with the name <driver>.layer.properties''' ###This should eb a UTF8 write, but only works as ASCII #open(self.filename,'w').write(str(res)) with codecs.open(self.lcfilename, 'w', 'utf-8') as lconf: lconf.write(res) self._readConfigFile(self.lcfilename) def _readConfigFile(self, fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed try: with codecs.open(fn, 'r', 'utf-8') as cf: self.cp.readfp(cf) except ParsingError as pe: ldslog.error( '{0} file corrupt. Please correct the error; {1} OR delete and rebuild' .format(fn, str(pe))) raise @override(LayerReader) def findLayerIdByName(self, name): '''Reverse lookup of section by associated name, finds first occurance only''' lid = filter(lambda n: name == self.cp.get(n, 'name'), self.cp.sections()) return lid[0] if len(lid) > 0 else None @override(LayerReader) def getLayerNames(self, refresh=False): '''Returns sections from properties file''' lcnames = [] for sec in self.cp.sections(): lcn = self.cp.get(sec, 'name') lcc = self.cp.get(sec, 'category') lcnames += [ (sec if type(sec) == unicode else unicode(sec, 'utf8'), lcn if type(lcn) == unicode else unicode(lcn, 'utf8'), (lcc if type(lcc) == unicode else unicode( lcc, 'utf8')).split(',')), ] return lcnames @override(LayerReader) def readLayerProperty(self, layer, key): try: if isinstance(layer, tuple) or isinstance(layer, list): value = () for l in layer: value += (LU.assessNone(self.cp.get(l, key)), ) else: value = LU.assessNone(self.cp.get(layer, key)) except: '''return a default value otherwise none which would also be a default for some keys''' #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed? return {'pkey': 'ID', 'name': layer, 'geocolumn': 'SHAPE'}.get(key) return value def _readSingleLayerProperty(self, layer, key): return LU.assessNone(self.cp.get(layer, key)) @override(LayerReader) def writeLayerProperty(self, layer, field, value): '''Write changes to layer config table''' #if value: value = value.strip() try: if (isinstance(layer, tuple) or isinstance(layer, list)) and ( isinstance(value, tuple) or isinstance(value, list)): for l, v in zip(layer, value): self.cp.set(l, field, v.strip() if v else '') else: self.cp.set(layer, field, value.strip() if value else '') with codecs.open(self.lcfilename, 'w', 'utf-8') as configfile: self.cp.write(configfile) #ldslog.debug("Check "+str(field)+" for layer "+str(layer)+" is set to "+str(value)+" : GetField="+self.cp.get(layer, field)) except Exception as e: ldslog.warn('Problem writing LM date to layer config file. ' + str(e)) @override(LayerReader) def readLayerParameters(self, id): #def readLayerSchemaConfig(self,layer): '''Full Layer config reader. Returns the config values for the whole layer or makes sensible guesses for defaults''' from LDSUtilities import LayerConfEntry # # try: # defn = self.cp.get(layer, 'sql') # #if the user has gone to the trouble of defining their own schema in SQL just return that # return (defn,None,None,None,None,None,None,None,None) # except: # pass '''optional but one way to record the type and name of a column is to save a string tuple (name,type) and parse this at build time''' try: pkey = self.cp.get(id, 'pkey') except NoOptionError: ldslog.warn( "LayerSchema: No Primary Key Column defined, default to 'ID'") pkey = 'ID' '''names are/can-be stored so we can reverse search by layer name''' try: name = self.cp.get(id, 'name') except NoOptionError: ldslog.warn( "LayerSchema: No Name saved in config for this layer, returning ID" ) name = id if name is None: name = id '''names are/can-be stored so we can reverse search by layer name''' try: group = LU.assessNone(self.cp.get(id, 'category')) except NoOptionError: ldslog.warn("Group List: No Groups defined for this layer") group = None if not group: pass try: gcol = self.cp.get(id, 'geocolumn') except NoOptionError: ldslog.warn( "LayerSchema: No Geo Column defined, default to 'SHAPE'") gcol = 'SHAPE' #i dont think we need this anymore using the new logic, if gcol->spatial, if pkey->unique # try: # index = self.cp.get(layer, 'index') # except NoOptionError: # ldslog.warn("LayerSchema: No Index Column/Specification defined, default to None") # index = None try: epsg = self.cp.get(id, 'epsg') except NoOptionError: #print "No Projection Transformation defined"#don't really need to state the default occurance epsg = None try: lmod = self.cp.get(id, 'lastmodified') except NoOptionError: ldslog.warn( "LayerSchema: No Last-Modified date recorded, successful update will write current time here" ) lmod = None try: disc = self.cp.get(id, 'discard') except NoOptionError: disc = None try: cql = self.cp.get(id, 'cql') except NoOptionError: cql = None return LayerConfEntry(id, pkey, name, group, gcol, epsg, lmod, disc, cql) @override(LayerReader) def readAllLayerParameters(self): '''Gets all LC entries as a list of LCEs using readLayerParameters''' lcel = [] for ln in self.getLayerNames(): lcel += [ self.readLayerParameters(ln), ] return lcel
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', } } # A mapping of old default values that we want to change and warn the user # about. Mapping of section -> setting -> { old, replace, by_version } deprecated_values = { 'core': { 'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'), }, } def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): if (self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif (self.getboolean("webserver", "authenticate") and self.get( "webserver", "owner_mode") not in ['user', 'ldapgroup']): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif (self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ('airflow.contrib.auth.backends.ldap_auth')): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): old, new, version = info if self.get(section, name, fallback=None) == old: # Make sure the env var option is removed, otherwise it # would be read and used instead of the value we set env_var = self._env_var_name(section, name) os.environ.pop(env_var, None) self.set(section, name, new) warnings.warn( 'The {name} setting in [{section}] has the old default value ' 'of {old!r}. This value has been changed to {new!r} in the ' 'running config, but please update your config before Apache ' 'Airflow {version}.'.format(name=name, section=section, old=old, new=new, version=version), FutureWarning) self.is_validated = True @staticmethod def _env_var_name(section, key): return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var( super(AirflowConfigParser, self).get(section, deprecated_name, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning("section/key [%s/%s] not found in config", section, key) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(section=section, key=key)) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super(AirflowConfigParser, self).read(filenames, **kwargs) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except NoOptionError: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.'.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend') } def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False def _validate(self): if ( self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option # ...then the default config if self.defaults.has_option(section, key): return expand_env_var( self.defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format(**locals()) ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.defaults.has_option(section, option) and remove_default: self.defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if section not in self._sections and section not in self.defaults._sections: return None _section = copy.deepcopy(self.defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg' or 'default'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool """ cfg = copy.deepcopy(self.defaults._sections) cfg.update(copy.deepcopy(self._sections)) # remove __name__ (affects Python 2 only) for options in cfg.values(): options.pop('__name__', None) # add source if display_source: for section in cfg: for k, v in cfg[section].items(): cfg[section][k] = (v, 'airflow config') # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: opt = None if opt: if ( not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update( {key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'bash cmd') cfg.setdefault(section, OrderedDict()).update({key: opt}) return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.defaults.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE)
class LayerFileReader(LayerReader): def __init__(self,lcfname): ''' Constructor ''' super(LayerFileReader,self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename) def isCurrent(self): '''TF test to decide whether to init''' return self.lcfilename and self._fileexists() and len(self.cp.sections())>0 def close(self): self.cp = None self.lcfilename = None self.lcfname = None def _fileexists(self): return os.path.exists(self.lcfilename) def buildConfigLayer(self,res): '''Just write a file in conf with the name <driver>.layer.properties''' ###This should eb a UTF8 write, but only works as ASCII #open(self.filename,'w').write(str(res)) with codecs.open(self.lcfilename,'w','utf-8') as lconf: lconf.write(res) self._readConfigFile(self.lcfilename) def _readConfigFile(self,fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed try: with codecs.open(fn,'r','utf-8') as cf: self.cp.readfp(cf) except ParsingError as pe: ldslog.error('{0} file corrupt. Please correct the error; {1} OR delete and rebuild'.format(fn,str(pe))) raise @override(LayerReader) def findLayerIdByName(self,name): '''Reverse lookup of section by associated name, finds first occurance only''' lid = filter(lambda n: name==self.cp.get(n,'name'),self.cp.sections()) return lid[0] if len(lid)>0 else None @override(LayerReader) def getLayerNames(self,refresh=False): '''Returns sections from properties file''' lcnames = [] for sec in self.cp.sections(): lcn = self.cp.get(sec, 'name') lcc = self.cp.get(sec,'category') lcnames += [(sec if type(sec)==unicode else unicode(sec,'utf8'), lcn if type(lcn)==unicode else unicode(lcn,'utf8'), (lcc if type(lcc)==unicode else unicode(lcc,'utf8')).split(',')),] return lcnames @override(LayerReader) def readLayerProperty(self,layer,key): try: if isinstance(layer,tuple) or isinstance(layer,list): value = () for l in layer: value += (LU.assessNone(self.cp.get(l, key)),) else: value = LU.assessNone(self.cp.get(layer, key)) except: '''return a default value otherwise none which would also be a default for some keys''' #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed? return {'pkey':'ID','name':layer,'geocolumn':'SHAPE'}.get(key) return value def _readSingleLayerProperty(self,layer,key): return LU.assessNone(self.cp.get(layer, key)) @override(LayerReader) def writeLayerProperty(self,layer,field,value): '''Write changes to layer config table''' #if value: value = value.strip() try: if (isinstance(layer,tuple) or isinstance(layer,list)) and (isinstance(value,tuple) or isinstance(value,list)): for l,v in zip(layer,value): self.cp.set(l,field,v.strip() if v else '') else: self.cp.set(layer,field,value.strip() if value else '') with codecs.open(self.lcfilename, 'w','utf-8') as configfile: self.cp.write(configfile) #ldslog.debug("Check "+str(field)+" for layer "+str(layer)+" is set to "+str(value)+" : GetField="+self.cp.get(layer, field)) except Exception as e: ldslog.warn('Problem writing LM date to layer config file. '+str(e)) @override(LayerReader) def readLayerParameters(self,id): #def readLayerSchemaConfig(self,layer): '''Full Layer config reader. Returns the config values for the whole layer or makes sensible guesses for defaults''' from LDSUtilities import LayerConfEntry # # try: # defn = self.cp.get(layer, 'sql') # #if the user has gone to the trouble of defining their own schema in SQL just return that # return (defn,None,None,None,None,None,None,None,None) # except: # pass '''optional but one way to record the type and name of a column is to save a string tuple (name,type) and parse this at build time''' try: pkey = self.cp.get(id, 'pkey') except NoOptionError: ldslog.warn("LayerSchema: No Primary Key Column defined, default to 'ID'") pkey = 'ID' '''names are/can-be stored so we can reverse search by layer name''' try: name = self.cp.get(id, 'name') except NoOptionError: ldslog.warn("LayerSchema: No Name saved in config for this layer, returning ID") name = id if name is None: name = id '''names are/can-be stored so we can reverse search by layer name''' try: group = LU.assessNone(self.cp.get(id, 'category')) except NoOptionError: ldslog.warn("Group List: No Groups defined for this layer") group = None if not group: pass try: gcol = self.cp.get(id, 'geocolumn') except NoOptionError: ldslog.warn("LayerSchema: No Geo Column defined, default to 'SHAPE'") gcol = 'SHAPE' #i dont think we need this anymore using the new logic, if gcol->spatial, if pkey->unique # try: # index = self.cp.get(layer, 'index') # except NoOptionError: # ldslog.warn("LayerSchema: No Index Column/Specification defined, default to None") # index = None try: epsg = self.cp.get(id, 'epsg') except NoOptionError: #print "No Projection Transformation defined"#don't really need to state the default occurance epsg = None try: lmod = self.cp.get(id, 'lastmodified') except NoOptionError: ldslog.warn("LayerSchema: No Last-Modified date recorded, successful update will write current time here") lmod = None try: disc = self.cp.get(id, 'discard') except NoOptionError: disc = None try: cql = self.cp.get(id, 'cql') except NoOptionError: cql = None return LayerConfEntry(id,pkey,name,group,gcol,epsg,lmod,disc,cql) @override(LayerReader) def readAllLayerParameters(self): '''Gets all LC entries as a list of LCEs using readLayerParameters''' lcel = [] for ln in self.getLayerNames(): lcel += [self.readLayerParameters(ln),] return lcel
def _readConfigFile(self, fn): '''Reads named config file''' #Split off so you can override the config file on the same reader object if needed self.cp = ConfigParser() with codecs.open(fn, 'r', 'utf-8') as cf: self.cp.readfp(cf)
def __init__(self, default_config=None, *args, **kwargs): super().__init__(*args, **kwargs) self.spacetimegis_defaults = ConfigParser(*args, **kwargs) if not default_config: self.spacetimegis_defaults.read_string(default_config)