class ExcaliburConfigParser(ConfigParser): def __init__(self, default_config=None, *args, **kwargs): super(ExcaliburConfigParser, self).__init__(*args, **kwargs) self.excalibur_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.excalibur_defaults.read_string(default_config) self.is_validated = False def _validate(self): if self.get( "core", "executor") != "SequentialExecutor" and "sqlite" in self.get( "core", "sql_alchemy_conn"): raise ValueError("Cannot use sqlite with the {}".format( self.get("core", "executor"))) self.is_validated = True def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() if super(ExcaliburConfigParser, self).has_option(section, key): return expand_env_var( super(ExcaliburConfigParser, self).get(section, key, **kwargs)) if self.excalibur_defaults.has_option(section, key): return expand_env_var( self.excalibur_defaults.get(section, key, **kwargs)) else: raise ValueError("section/key [{section}/{key}] not found in" " config".format(**locals())) def read(self, filename): super(ExcaliburConfigParser, self).read(filename) self._validate()
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', } } # A mapping of old default values that we want to change and warn the user # about. Mapping of section -> setting -> { old, replace, by_version } deprecated_values = { 'core': { 'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'), }, } def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): if (self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif (self.getboolean("webserver", "authenticate") and self.get( "webserver", "owner_mode") not in ['user', 'ldapgroup']): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif (self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ('airflow.contrib.auth.backends.ldap_auth')): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): old, new, version = info if self.get(section, name, fallback=None) == old: # Make sure the env var option is removed, otherwise it # would be read and used instead of the value we set env_var = self._env_var_name(section, name) os.environ.pop(env_var, None) self.set(section, name, new) warnings.warn( 'The {name} setting in [{section}] has the old default value ' 'of {old!r}. This value has been changed to {new!r} in the ' 'running config, but please update your config before Apache ' 'Airflow {version}.'.format(name=name, section=section, old=old, new=new, version=version), FutureWarning) self.is_validated = True @staticmethod def _env_var_name(section, key): return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var( super(AirflowConfigParser, self).get(section, deprecated_name, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning("section/key [%s/%s] not found in config", section, key) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(section=section, key=key)) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super(AirflowConfigParser, self).read(filenames, **kwargs) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except NoOptionError: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.'.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', } } deprecation_format_string = ( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.' ) def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): if ( self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var(super(AirflowConfigParser, self).get( section, deprecated_name, **kwargs )) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key): return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format(**locals()) ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict( self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update( {key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( self.deprecation_format_string.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. # These configs can also be fetched from Secrets backend # following the "{section}__{name}__secret" pattern sensitive_config_values = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'flower_basic_auth'), ('celery', 'result_backend'), # Todo: remove this in Airflow 1.11 ('celery', 'celery_result_backend'), ('atlas', 'password'), ('smtp', 'smtp_password'), ('ldap', 'bind_password'), ('kubernetes', 'git_password'), } # A two-level mapping of (section -> new_name -> old_name). When reading # new_name, the old_name will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old name will be used instead deprecated_options = { 'celery': { # Remove these keys in Airflow 1.11 'worker_concurrency': 'celeryd_concurrency', 'result_backend': 'celery_result_backend', 'broker_url': 'celery_broker_url', 'ssl_active': 'celery_ssl_active', 'ssl_cert': 'celery_ssl_cert', 'ssl_key': 'celery_ssl_key', }, 'elasticsearch': { 'host': 'elasticsearch_host', 'log_id_template': 'elasticsearch_log_id_template', 'end_of_log_mark': 'elasticsearch_end_of_log_mark', 'frontend': 'elasticsearch_frontend', 'write_stdout': 'elasticsearch_write_stdout', 'json_format': 'elasticsearch_json_format', 'json_fields': 'elasticsearch_json_fields' } } # A mapping of old default values that we want to change and warn the user # about. Mapping of section -> setting -> { old, replace, by_version } deprecated_values = { 'core': { 'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'), }, } # This method transforms option names on every read, get, or set operation. # This changes from the default behaviour of ConfigParser from lowercasing # to instead be case-preserving def optionxform(self, optionstr): return optionstr def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.airflow_defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.airflow_defaults.read_string(default_config) self.is_validated = False def _validate(self): self._validate_config_dependencies() for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): old, new, version = info if self.get(section, name, fallback=None) == old: # Make sure the env var option is removed, otherwise it # would be read and used instead of the value we set env_var = self._env_var_name(section, name) os.environ.pop(env_var, None) self.set(section, name, new) warnings.warn( 'The {name} setting in [{section}] has the old default value ' 'of {old!r}. This value has been changed to {new!r} in the ' 'running config, but please update your config before Apache ' 'Airflow {version}.'.format( name=name, section=section, old=old, new=new, version=version ), FutureWarning ) self.is_validated = True def _validate_config_dependencies(self): """ Validate that config values aren't invalid given other config values or system-level limitations and requirements. """ if ( self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") if self.has_option('core', 'mp_start_method'): mp_start_method = self.get('core', 'mp_start_method') start_method_options = multiprocessing.get_all_start_methods() if mp_start_method not in start_method_options: raise AirflowConfigException( "mp_start_method should not be " + mp_start_method + ". Possible values are " + ", ".join(start_method_options)) @staticmethod def _env_var_name(section, key): return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) # alternatively AIRFLOW__{SECTION}__{KEY}_CMD (for a command) env_var_cmd = env_var + '_CMD' if env_var_cmd in os.environ: # if this is a valid command key... if (section, key) in self.sensitive_config_values: return run_command(os.environ[env_var_cmd]) # alternatively AIRFLOW__{SECTION}__{KEY}_SECRET (to get from Secrets Backend) env_var_secret_path = env_var + '_SECRET' if env_var_secret_path in os.environ: # if this is a valid secret path... if (section, key) in self.sensitive_config_values: return _get_config_value_from_secret_backend(os.environ[env_var_secret_path]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.sensitive_config_values: if super(AirflowConfigParser, self).has_option(section, fallback_key): command = super(AirflowConfigParser, self).get(section, fallback_key) return run_command(command) def _get_secret_option(self, section, key): """Get Config option values from Secret Backend""" fallback_key = key + '_secret' # if this is a valid secret key... if (section, key) in self.sensitive_config_values: if super(AirflowConfigParser, self).has_option(section, fallback_key): secrets_path = super(AirflowConfigParser, self).get(section, fallback_key) return _get_config_value_from_secret_backend(secrets_path) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() deprecated_name = self.deprecated_options.get(section, {}).get(key, None) # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option if deprecated_name: option = self._get_env_var_option(section, deprecated_name) if option is not None: self._warn_deprecate(section, key, deprecated_name) return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) if deprecated_name: if super(AirflowConfigParser, self).has_option(section, deprecated_name): self._warn_deprecate(section, key, deprecated_name) return expand_env_var(super(AirflowConfigParser, self).get( section, deprecated_name, **kwargs )) # ...then commands option = self._get_cmd_option(section, key) if option: return option if deprecated_name: option = self._get_cmd_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then from secret backends option = self._get_secret_option(section, key) if option: return option if deprecated_name: option = self._get_secret_option(section, deprecated_name) if option: self._warn_deprecate(section, key, deprecated_name) return option # ...then the default config if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.airflow_defaults.get(section, key, **kwargs)) else: log.warning( "section/key [%s/%s] not found in config", section, key ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(section=section, key=key)) def getimport(self, section, key, **kwargs): """ Reads options, imports the full qualified name, and returns the object. In case of failure, it throws an exception a clear message with the key aad the section names :return: The object or None, if the option is empty """ full_qualified_path = conf.get(section=section, key=key, **kwargs) if not full_qualified_path: return None try: return import_string(full_qualified_path) except ImportError as e: log.error(e) raise AirflowConfigException( 'The object could not be loaded. Please check "{key}" key in "{section}" section. ' 'Current value: "{full_qualified_path}".'.format( key=key, section=section, full_qualified_path=full_qualified_path) ) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: raise ValueError( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super(AirflowConfigParser, self).read(filenames, **kwargs) self._validate() def read_dict(self, *args, **kwargs): super(AirflowConfigParser, self).read_dict(*args, **kwargs) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except (NoOptionError, NoSectionError): return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.airflow_defaults.has_option(section, option) and remove_default: self.airflow_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.airflow_defaults._sections): return None _section = copy.deepcopy(self.airflow_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '') if key.endswith("_CMD"): key = key[:-4] key = key.lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def write(self, fp, space_around_delimiters=True): # This is based on the configparser.RawConfigParser.write method code to add support for # reading options from environment variables. if space_around_delimiters: d = " {} ".format(self._delimiters[0]) # type: ignore else: d = self._delimiters[0] # type: ignore if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) # type: ignore for section in self._sections: self._write_section(fp, section, self.getsection(section).items(), d) # type: ignore def as_dict( self, display_source=False, display_sensitive=False, raw=False, include_env=True, include_cmds=True, include_secret=True ): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool :param include_env: Should the value of configuration from AIRFLOW__ environment variables be included or not :type include_env: bool :param include_cmds: Should the result of calling any *_cmd config be set (True, default), or should the _cmd options be left as the command to run (False) :type include_cmds: bool :param include_secret: Should the result of calling any *_secret config be set (True, default), or should the _secret options be left as the path to get the secret from (False) :type include_secret: bool :return: Dictionary, where the key is the name of the section and the content is the dictionary with the name of the parameter and its value. """ cfg = {} configs = [ ('default', self.airflow_defaults), ('airflow.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k] = val # add env vars and overwrite because they have priority if include_env: for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__', 2) opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') section = section.lower() # if we lower key for kubernetes_environment_variables section, # then we won't be able to set any Airflow environment # variables. Airflow only parse environment variables starts # with AIRFLOW_. Therefore, we need to make it a special case. if section != 'kubernetes_environment_variables': key = key.lower() cfg.setdefault(section, OrderedDict()).update({key: opt}) # add bash commands if include_cmds: for (section, key) in self.sensitive_config_values: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'cmd') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_cmd'] # add config from secret backends if include_secret: for (section, key) in self.sensitive_config_values: opt = self._get_secret_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'secret') elif raw: opt = opt.replace('%', '%%') cfg.setdefault(section, OrderedDict()).update({key: opt}) del cfg[section][key + '_secret'] return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults log.info("Overriding settings with defaults from %s", DEFAULT_CONFIG_FILE_PATH) self.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config log.info("Reading default test configuration from %s", TEST_CONFIG_FILE_PATH) self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings log.info("Reading test configuration from %s", TEST_CONFIG_FILE) self.read(TEST_CONFIG_FILE) def _warn_deprecate(self, section, key, deprecated_name): warnings.warn( 'The {old} option in [{section}] has been renamed to {new} - the old ' 'setting has been used, but please update your config.'.format( old=deprecated_name, new=key, section=section, ), DeprecationWarning, stacklevel=3, )
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = {('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend')} def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False def _validate(self): if (self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif (self.getboolean("webserver", "authenticate") and self.get( "webserver", "owner_mode") not in ['user', 'ldapgroup']): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif (self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ('airflow.contrib.auth.backends.ldap_auth')): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option # ...then the default config if self.defaults.has_option(section, key): return expand_env_var(self.defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format( **locals())) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.defaults.has_option(section, option) and remove_default: self.defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if section not in self._sections and section not in self.defaults._sections: return None _section = copy.deepcopy(self.defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg' or 'default'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool """ cfg = copy.deepcopy(self.defaults._sections) cfg.update(copy.deepcopy(self._sections)) # remove __name__ (affects Python 2 only) for options in cfg.values(): options.pop('__name__', None) # add source if display_source: for section in cfg: for k, v in cfg[section].items(): cfg[section][k] = (v, 'airflow config') # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: opt = None if opt: if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'bash cmd') cfg.setdefault(section, OrderedDict()).update({key: opt}) return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.defaults.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE)
class SpacetimeGISConfigParser(ConfigParser): def __init__(self, default_config=None, *args, **kwargs): super().__init__(*args, **kwargs) self.spacetimegis_defaults = ConfigParser(*args, **kwargs) if not default_config: self.spacetimegis_defaults.read_string(default_config) @staticmethod def _env_var_name(section, key): return 'SPACETIMEGIS__{S}__{K}'.format(S=section.upper(), K=key.upper()) def _get_env_var_option(self, section, key): # must have format SPACETIMEGIS__{SECTION}__{KEY} (note double underscore) env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super().has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var(super().get(section, key, **kwargs)) # ...then the default config if self.spacetimegis_defaults.has_option(section, key) or 'fallback' in kwargs: return expand_env_var( self.spacetimegis_defaults.get(section, key, **kwargs)) def getboolean(self, section, key, **kwargs): val = str(self.get(section, key, **kwargs)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val in ('t', 'true', '1'): return True elif val in ('f', 'false', '0'): return False else: # logger.writelog(LogLevel.error, # 'The value for configuration option "{}:{}" is not a ' # 'boolean (received "{}").'.format(section, key, val)) raise ValueError( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key, **kwargs): return int(self.get(section, key, **kwargs)) def getfloat(self, section, key, **kwargs): return float(self.get(section, key, **kwargs)) def read(self, filenames, **kwargs): super().read(filenames, **kwargs) def read_dict(self, *args, **kwargs): super().read_dict(*args, **kwargs) def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) # UNSET to avoid logging a warning about missing values self.get(section, option, fallback=_UNSET) return True except NoOptionError: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super().has_option(section, option): super().remove_option(section, option) if self.spacetimegis_defaults.has_option(section, option) and remove_default: self.spacetimegis_defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :rtype: dict """ if (section not in self._sections and section not in self.spacetimegis_defaults._sections): return None _section = copy.deepcopy(self.spacetimegis_defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) section_prefix = 'SPACETIMEGIS__{S}__'.format(S=section.upper()) for env_var in sorted(os.environ.keys()): if env_var.startswith(section_prefix): key = env_var.replace(section_prefix, '').lower() _section[key] = self._get_env_var_option(section, key) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False, raw=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg', 'default', 'env var', or 'cmd'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool :param raw: Should the values be output as interpolated values, or the "raw" form that can be fed back in to ConfigParser :type raw: bool """ cfg = {} configs = [ ('default', self.spacetimegis_defaults), ('spacetimegis.cfg', self), ] for (source_name, config) in configs: for section in config.sections(): sect = cfg.setdefault(section, OrderedDict()) for (k, val) in config.items(section=section, raw=raw): if display_source: val = (val, source_name) sect[k.upper()] = val # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('SPACETIMEGIS__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: continue if not display_sensitive and ev != 'SPACETIMEGIS__CORE__UNIT_TEST_MODE': opt = '< hidden >' elif raw: opt = opt.replace('%', '%%') if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update({key.lower(): opt}) return cfg def as_all_dict(self, display_source=False, display_sensitive=False, raw=False): tmp = self.as_dict(display_source, display_sensitive, raw) all_cfg = {} for val in tmp.values(): if not all_cfg: all_cfg = val.copy() else: all_cfg.update(val) return all_cfg
class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend') } def __init__(self, default_config=None, *args, **kwargs): super(AirflowConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False def _validate(self): if ( self.get("core", "executor") != 'SequentialExecutor' and "sqlite" in self.get('core', 'sql_alchemy_conn')): raise AirflowConfigException( "error: cannot use sqlite with the {}".format( self.get('core', 'executor'))) elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode") not in ['user', 'ldapgroup'] ): raise AirflowConfigException( "error: owner_mode option should be either " "'user' or 'ldapgroup' when filtering by owner is set") elif ( self.getboolean("webserver", "authenticate") and self.get("webserver", "owner_mode").lower() == 'ldapgroup' and self.get("webserver", "auth_backend") != ( 'airflow.contrib.auth.backends.ldap_auth') ): raise AirflowConfigException( "error: attempt at using ldapgroup " "filtering without using the Ldap backend") self.is_validated = True def _get_env_var_option(self, section, key): # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore) env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if super(AirflowConfigParser, self) \ .has_option(section, fallback_key): command = super(AirflowConfigParser, self) \ .get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # first check environment variables option = self._get_env_var_option(section, key) if option is not None: return option # ...then the config file if super(AirflowConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(AirflowConfigParser, self).get(section, key, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option # ...then the default config if self.defaults.has_option(section, key): return expand_env_var( self.defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format(**locals()) ) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise AirflowConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): super(AirflowConfigParser, self).read(filenames) self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except AirflowConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(AirflowConfigParser, self).has_option(section, option): super(AirflowConfigParser, self).remove_option(section, option) if self.defaults.has_option(section, option) and remove_default: self.defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if section not in self._sections and section not in self.defaults._sections: return None _section = copy.deepcopy(self.defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'airflow.cfg' or 'default'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool """ cfg = copy.deepcopy(self.defaults._sections) cfg.update(copy.deepcopy(self._sections)) # remove __name__ (affects Python 2 only) for options in cfg.values(): options.pop('__name__', None) # add source if display_source: for section in cfg: for k, v in cfg[section].items(): cfg[section][k] = (v, 'airflow config') # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: opt = None if opt: if ( not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'): opt = '< hidden >' if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update( {key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'bash cmd') cfg.setdefault(section, OrderedDict()).update({key: opt}) return cfg def load_test_config(self): """ Load the unit test configuration. Note: this is not reversible. """ # override any custom settings with defaults self.defaults.read_string(parameterized_config(DEFAULT_CONFIG)) # then read test config self.read_string(parameterized_config(TEST_CONFIG)) # then read any "custom" test settings self.read(TEST_CONFIG_FILE)
class XToolConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. as_command_stdout = { ('core', 'sql_alchemy_conn'), ('core', 'fernet_key'), ('celery', 'broker_url'), ('celery', 'result_backend'), } def __init__(self, default_config=None, *args, **kwargs): super(XToolConfigParser, self).__init__(*args, **kwargs) self.defaults = ConfigParser(*args, **kwargs) # 读取配置字符串 if default_config is not None: self.defaults.read_string(default_config) self.is_validated = False def _validate(self): self.is_validated = True def _get_env_var_option(self, section, key): """把环境变量的值中包含的”~”和”~user”转换成用户目录,并获取配置结果值 .""" # must have format XTOOL__{SECTION}__{KEY} (note double underscore) env_var = 'XTOOL__{S}__{K}'.format(S=section.upper(), K=key.upper()) if env_var in os.environ: return expand_env_var(os.environ[env_var]) def _get_cmd_option(self, section, key): """从配置项中获取指令,并执行指令获取指令执行后的返回值 - 如果key不存在_cmd结尾,则获取key的值 - 如果key没有配置 且 key以_cmd结尾,则获取key的值,并执行值表示的表达式,返回表达式的结果 """ fallback_key = key + '_cmd' # if this is a valid command key... if (section, key) in self.as_command_stdout: if self.has_option(section, fallback_key): command = self.get(section, fallback_key) return run_command(command) def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() # 首先从环境变量中获取配置值,如果环境变量中存在,则不再从配置文件中获取 option = self._get_env_var_option(section, key) if option is not None: return option # 然后从配置文件中获取 if super(XToolConfigParser, self).has_option(section, key): # Use the parent's methods to get the actual config here to be able to # separate the config from default config. return expand_env_var( super(XToolConfigParser, self).get(section, key, **kwargs)) # 执行表达式,获取结果 option = self._get_cmd_option(section, key) if option: return option # ...then the default config if self.defaults.has_option(section, key): return expand_env_var( self.defaults.get(section, key, **kwargs)) else: log.warning( "section/key [{section}/{key}] not found in config".format(**locals()) ) # 配置不存在,抛出异常 raise XToolConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals())) def getboolean(self, section, key): val = str(self.get(section, key)).lower().strip() # 去掉结尾的注释 if '#' in val: val = val.split('#')[0].strip() if val.lower() in ('t', 'true', '1'): return True elif val.lower() in ('f', 'false', '0'): return False else: raise XToolConfigException( 'The value for configuration option "{}:{}" is not a ' 'boolean (received "{}").'.format(section, key, val)) def getint(self, section, key): return int(self.get(section, key)) def getfloat(self, section, key): return float(self.get(section, key)) def read(self, filenames): """读取多个配置文件,并进行校验 @note: 执行此函数,获取的配置项会覆盖掉构造函数中的默认配置 default_config """ super(XToolConfigParser, self).read(filenames, encoding="utf-8") self._validate() def has_option(self, section, option): try: # Using self.get() to avoid reimplementing the priority order # of config variables (env, config, cmd, defaults) self.get(section, option) return True except XToolConfigException: return False def remove_option(self, section, option, remove_default=True): """ Remove an option if it exists in config from a file or default config. If both of config have the same option, this removes the option in both configs unless remove_default=False. """ if super(XToolConfigParser, self).has_option(section, option): super(XToolConfigParser, self).remove_option(section, option) if self.defaults.has_option(section, option) and remove_default: self.defaults.remove_option(section, option) def getsection(self, section): """ Returns the section as a dict. Values are converted to int, float, bool as required. :param section: section from the config :return: dict """ if section not in self._sections and section not in self.defaults._sections: return None _section = copy.deepcopy(self.defaults._sections[section]) if section in self._sections: _section.update(copy.deepcopy(self._sections[section])) # 遍历section下所有的key,对value进行格式化处理 for key, val in iteritems(_section): try: val = int(val) except ValueError: try: val = float(val) except ValueError: if val.lower() in ('t', 'true'): val = True elif val.lower() in ('f', 'false'): val = False _section[key] = val return _section def as_dict(self, display_source=False, display_sensitive=False): """ Returns the current configuration as an OrderedDict of OrderedDicts. :param display_source: If False, the option value is returned. If True, a tuple of (option_value, source) is returned. Source is either 'xTool.cfg' or 'default'. :type display_source: bool :param display_sensitive: If True, the values of options set by env vars and bash commands will be displayed. If False, those options are shown as '< hidden >' :type display_sensitive: bool """ cfg = copy.deepcopy(self.defaults._sections) cfg.update(copy.deepcopy(self._sections)) # remove __name__ (affects Python 2 only) for options in cfg.values(): options.pop('__name__', None) # add source if display_source: for section in cfg: for k, v in cfg[section].items(): cfg[section][k] = (v, 'xTool config') # add env vars and overwrite because they have priority for ev in [ev for ev in os.environ if ev.startswith('XTOOL__')]: try: _, section, key = ev.split('__') opt = self._get_env_var_option(section, key) except ValueError: opt = None if opt: if ( not display_sensitive and ev != 'XTOOL__CORE__UNIT_TEST_MODE'): opt = '< hidden >' if display_source: opt = (opt, 'env var') cfg.setdefault(section.lower(), OrderedDict()).update( {key.lower(): opt}) # add bash commands for (section, key) in self.as_command_stdout: opt = self._get_cmd_option(section, key) if opt: if not display_sensitive: opt = '< hidden >' if display_source: opt = (opt, 'bash cmd') cfg.setdefault(section, OrderedDict()).update({key: opt}) return cfg