class ExcaliburConfigParser(ConfigParser):
def __init__(self, default_config=None, *args, **kwargs):
super(ExcaliburConfigParser, self).__init__(*args, **kwargs)
self.excalibur_defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.excalibur_defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
if self.get(
"core",
"executor") != "SequentialExecutor" and "sqlite" in self.get(
"core", "sql_alchemy_conn"):
raise ValueError("Cannot use sqlite with the {}".format(
self.get("core", "executor")))
self.is_validated = True
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
if super(ExcaliburConfigParser, self).has_option(section, key):
return expand_env_var(
super(ExcaliburConfigParser, self).get(section, key, **kwargs))
if self.excalibur_defaults.has_option(section, key):
return expand_env_var(
self.excalibur_defaults.get(section, key, **kwargs))
else:
raise ValueError("section/key [{section}/{key}] not found in"
" config".format(**locals()))
def read(self, filename):
super(ExcaliburConfigParser, self).read(filename)
self._validate()
class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
as_command_stdout = {
('core', 'sql_alchemy_conn'),
('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'result_backend'),
# Todo: remove this in Airflow 1.11
('celery', 'celery_result_backend'),
('atlas', 'password'),
('smtp', 'smtp_password'),
('ldap', 'bind_password'),
('kubernetes', 'git_password'),
}
# A two-level mapping of (section -> new_name -> old_name). When reading
# new_name, the old_name will be checked to see if it exists. If it does a
# DeprecationWarning will be issued and the old name will be used instead
deprecated_options = {
'celery': {
# Remove these keys in Airflow 1.11
'worker_concurrency': 'celeryd_concurrency',
'result_backend': 'celery_result_backend',
'broker_url': 'celery_broker_url',
'ssl_active': 'celery_ssl_active',
'ssl_cert': 'celery_ssl_cert',
'ssl_key': 'celery_ssl_key',
}
}
# A mapping of old default values that we want to change and warn the user
# about. Mapping of section -> setting -> { old, replace, by_version }
deprecated_values = {
'core': {
'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
},
}
def __init__(self, default_config=None, *args, **kwargs):
super(AirflowConfigParser, self).__init__(*args, **kwargs)
self.airflow_defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.airflow_defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
if (self.get("core", "executor") != 'SequentialExecutor'
and "sqlite" in self.get('core', 'sql_alchemy_conn')):
raise AirflowConfigException(
"error: cannot use sqlite with the {}".format(
self.get('core', 'executor')))
elif (self.getboolean("webserver", "authenticate") and self.get(
"webserver", "owner_mode") not in ['user', 'ldapgroup']):
raise AirflowConfigException(
"error: owner_mode option should be either "
"'user' or 'ldapgroup' when filtering by owner is set")
elif (self.getboolean("webserver", "authenticate")
and self.get("webserver", "owner_mode").lower() == 'ldapgroup'
and self.get("webserver", "auth_backend") !=
('airflow.contrib.auth.backends.ldap_auth')):
raise AirflowConfigException(
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
old, new, version = info
if self.get(section, name, fallback=None) == old:
# Make sure the env var option is removed, otherwise it
# would be read and used instead of the value we set
env_var = self._env_var_name(section, name)
os.environ.pop(env_var, None)
self.set(section, name, new)
warnings.warn(
'The {name} setting in [{section}] has the old default value '
'of {old!r}. This value has been changed to {new!r} in the '
'running config, but please update your config before Apache '
'Airflow {version}.'.format(name=name,
section=section,
old=old,
new=new,
version=version),
FutureWarning)
self.is_validated = True
@staticmethod
def _env_var_name(section, key):
return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
def _get_env_var_option(self, section, key):
# must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
env_var = self._env_var_name(section, key)
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.as_command_stdout:
if super(AirflowConfigParser, self) \
.has_option(section, fallback_key):
command = super(AirflowConfigParser, self) \
.get(section, fallback_key)
return run_command(command)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
deprecated_name = self.deprecated_options.get(section,
{}).get(key, None)
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
if deprecated_name:
option = self._get_env_var_option(section, deprecated_name)
if option is not None:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the config file
if super(AirflowConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(AirflowConfigParser, self).get(section, key, **kwargs))
if deprecated_name:
if super(AirflowConfigParser,
self).has_option(section, deprecated_name):
self._warn_deprecate(section, key, deprecated_name)
return expand_env_var(
super(AirflowConfigParser,
self).get(section, deprecated_name, **kwargs))
# ...then commands
option = self._get_cmd_option(section, key)
if option:
return option
if deprecated_name:
option = self._get_cmd_option(section, deprecated_name)
if option:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the default config
if self.airflow_defaults.has_option(section,
key) or 'fallback' in kwargs:
return expand_env_var(
self.airflow_defaults.get(section, key, **kwargs))
else:
log.warning("section/key [%s/%s] not found in config", section,
key)
raise AirflowConfigException(
"section/key [{section}/{key}] not found "
"in config".format(section=section, key=key))
def getboolean(self, section, key, **kwargs):
val = str(self.get(section, key, **kwargs)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val in ('t', 'true', '1'):
return True
elif val in ('f', 'false', '0'):
return False
else:
raise AirflowConfigException(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key, **kwargs):
return int(self.get(section, key, **kwargs))
def getfloat(self, section, key, **kwargs):
return float(self.get(section, key, **kwargs))
def read(self, filenames, **kwargs):
super(AirflowConfigParser, self).read(filenames, **kwargs)
self._validate()
def read_dict(self, *args, **kwargs):
super(AirflowConfigParser, self).read_dict(*args, **kwargs)
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
# UNSET to avoid logging a warning about missing values
self.get(section, option, fallback=_UNSET)
return True
except NoOptionError:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(AirflowConfigParser, self).has_option(section, option):
super(AirflowConfigParser, self).remove_option(section, option)
if self.airflow_defaults.has_option(section,
option) and remove_default:
self.airflow_defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:rtype: dict
"""
if (section not in self._sections
and section not in self.airflow_defaults._sections):
return None
_section = copy.deepcopy(self.airflow_defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
for env_var in sorted(os.environ.keys()):
if env_var.startswith(section_prefix):
key = env_var.replace(section_prefix, '').lower()
_section[key] = self._get_env_var_option(section, key)
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(self,
display_source=False,
display_sensitive=False,
raw=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg', 'default', 'env var', or 'cmd'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
:param raw: Should the values be output as interpolated values, or the
"raw" form that can be fed back in to ConfigParser
:type raw: bool
"""
cfg = {}
configs = [
('default', self.airflow_defaults),
('airflow.cfg', self),
]
for (source_name, config) in configs:
for section in config.sections():
sect = cfg.setdefault(section, OrderedDict())
for (k, val) in config.items(section=section, raw=raw):
if display_source:
val = (val, source_name)
sect[k] = val
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
continue
if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE':
opt = '< hidden >'
elif raw:
opt = opt.replace('%', '%%')
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(),
OrderedDict()).update({key.lower(): opt})
# add bash commands
for (section, key) in self.as_command_stdout:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'cmd')
elif raw:
opt = opt.replace('%', '%%')
cfg.setdefault(section, OrderedDict()).update({key: opt})
del cfg[section][key + '_cmd']
return cfg
def load_test_config(self):
"""
Load the unit test configuration.
Note: this is not reversible.
"""
# override any custom settings with defaults
self.read_string(parameterized_config(DEFAULT_CONFIG))
# then read test config
self.read_string(parameterized_config(TEST_CONFIG))
# then read any "custom" test settings
self.read(TEST_CONFIG_FILE)
def _warn_deprecate(self, section, key, deprecated_name):
warnings.warn(
'The {old} option in [{section}] has been renamed to {new} - the old '
'setting has been used, but please update your config.'.format(
old=deprecated_name,
new=key,
section=section,
),
DeprecationWarning,
stacklevel=3,
)
class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
as_command_stdout = {
('core', 'sql_alchemy_conn'),
('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'result_backend'),
# Todo: remove this in Airflow 1.11
('celery', 'celery_result_backend'),
('atlas', 'password'),
('smtp', 'smtp_password'),
('ldap', 'bind_password'),
('kubernetes', 'git_password'),
}
# A two-level mapping of (section -> new_name -> old_name). When reading
# new_name, the old_name will be checked to see if it exists. If it does a
# DeprecationWarning will be issued and the old name will be used instead
deprecated_options = {
'celery': {
# Remove these keys in Airflow 1.11
'worker_concurrency': 'celeryd_concurrency',
'result_backend': 'celery_result_backend',
'broker_url': 'celery_broker_url',
'ssl_active': 'celery_ssl_active',
'ssl_cert': 'celery_ssl_cert',
'ssl_key': 'celery_ssl_key',
}
}
deprecation_format_string = (
'The {old} option in [{section}] has been renamed to {new} - the old '
'setting has been used, but please update your config.'
)
def __init__(self, default_config=None, *args, **kwargs):
super(AirflowConfigParser, self).__init__(*args, **kwargs)
self.airflow_defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.airflow_defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
if (
self.get("core", "executor") != 'SequentialExecutor' and
"sqlite" in self.get('core', 'sql_alchemy_conn')):
raise AirflowConfigException(
"error: cannot use sqlite with the {}".format(
self.get('core', 'executor')))
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
):
raise AirflowConfigException(
"error: owner_mode option should be either "
"'user' or 'ldapgroup' when filtering by owner is set")
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
self.get("webserver", "auth_backend") != (
'airflow.contrib.auth.backends.ldap_auth')
):
raise AirflowConfigException(
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
self.is_validated = True
def _get_env_var_option(self, section, key):
# must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.as_command_stdout:
if super(AirflowConfigParser, self) \
.has_option(section, fallback_key):
command = super(AirflowConfigParser, self) \
.get(section, fallback_key)
return run_command(command)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
deprecated_name = self.deprecated_options.get(section, {}).get(key, None)
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
if deprecated_name:
option = self._get_env_var_option(section, deprecated_name)
if option is not None:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the config file
if super(AirflowConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(AirflowConfigParser, self).get(section, key, **kwargs))
if deprecated_name:
if super(AirflowConfigParser, self).has_option(section, deprecated_name):
self._warn_deprecate(section, key, deprecated_name)
return expand_env_var(super(AirflowConfigParser, self).get(
section,
deprecated_name,
**kwargs
))
# ...then commands
option = self._get_cmd_option(section, key)
if option:
return option
if deprecated_name:
option = self._get_cmd_option(section, deprecated_name)
if option:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the default config
if self.airflow_defaults.has_option(section, key):
return expand_env_var(
self.airflow_defaults.get(section, key, **kwargs))
else:
log.warning(
"section/key [{section}/{key}] not found in config".format(**locals())
)
raise AirflowConfigException(
"section/key [{section}/{key}] not found "
"in config".format(**locals()))
def getboolean(self, section, key):
val = str(self.get(section, key)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val.lower() in ('t', 'true', '1'):
return True
elif val.lower() in ('f', 'false', '0'):
return False
else:
raise AirflowConfigException(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key):
return int(self.get(section, key))
def getfloat(self, section, key):
return float(self.get(section, key))
def read(self, filenames):
super(AirflowConfigParser, self).read(filenames)
self._validate()
def read_dict(self, *args, **kwargs):
super(AirflowConfigParser, self).read_dict(*args, **kwargs)
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
self.get(section, option)
return True
except AirflowConfigException:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(AirflowConfigParser, self).has_option(section, option):
super(AirflowConfigParser, self).remove_option(section, option)
if self.airflow_defaults.has_option(section, option) and remove_default:
self.airflow_defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:return: dict
"""
if (section not in self._sections and
section not in self.airflow_defaults._sections):
return None
_section = copy.deepcopy(self.airflow_defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
for env_var in sorted(os.environ.keys()):
if env_var.startswith(section_prefix):
key = env_var.replace(section_prefix, '').lower()
_section[key] = self._get_env_var_option(section, key)
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(
self, display_source=False, display_sensitive=False, raw=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg', 'default', 'env var', or 'cmd'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
:param raw: Should the values be output as interpolated values, or the
"raw" form that can be fed back in to ConfigParser
:type raw: bool
"""
cfg = {}
configs = [
('default', self.airflow_defaults),
('airflow.cfg', self),
]
for (source_name, config) in configs:
for section in config.sections():
sect = cfg.setdefault(section, OrderedDict())
for (k, val) in config.items(section=section, raw=raw):
if display_source:
val = (val, source_name)
sect[k] = val
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
continue
if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
opt = '< hidden >'
elif raw:
opt = opt.replace('%', '%%')
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(), OrderedDict()).update(
{key.lower(): opt})
# add bash commands
for (section, key) in self.as_command_stdout:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'cmd')
elif raw:
opt = opt.replace('%', '%%')
cfg.setdefault(section, OrderedDict()).update({key: opt})
del cfg[section][key + '_cmd']
return cfg
def load_test_config(self):
"""
Load the unit test configuration.
Note: this is not reversible.
"""
# override any custom settings with defaults
self.read_string(parameterized_config(DEFAULT_CONFIG))
# then read test config
self.read_string(parameterized_config(TEST_CONFIG))
# then read any "custom" test settings
self.read(TEST_CONFIG_FILE)
def _warn_deprecate(self, section, key, deprecated_name):
warnings.warn(
self.deprecation_format_string.format(
old=deprecated_name,
new=key,
section=section,
),
DeprecationWarning,
stacklevel=3,
)
class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
# These configs can also be fetched from Secrets backend
# following the "{section}__{name}__secret" pattern
sensitive_config_values = {
('core', 'sql_alchemy_conn'),
('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'flower_basic_auth'),
('celery', 'result_backend'),
# Todo: remove this in Airflow 1.11
('celery', 'celery_result_backend'),
('atlas', 'password'),
('smtp', 'smtp_password'),
('ldap', 'bind_password'),
('kubernetes', 'git_password'),
}
# A two-level mapping of (section -> new_name -> old_name). When reading
# new_name, the old_name will be checked to see if it exists. If it does a
# DeprecationWarning will be issued and the old name will be used instead
deprecated_options = {
'celery': {
# Remove these keys in Airflow 1.11
'worker_concurrency': 'celeryd_concurrency',
'result_backend': 'celery_result_backend',
'broker_url': 'celery_broker_url',
'ssl_active': 'celery_ssl_active',
'ssl_cert': 'celery_ssl_cert',
'ssl_key': 'celery_ssl_key',
},
'elasticsearch': {
'host': 'elasticsearch_host',
'log_id_template': 'elasticsearch_log_id_template',
'end_of_log_mark': 'elasticsearch_end_of_log_mark',
'frontend': 'elasticsearch_frontend',
'write_stdout': 'elasticsearch_write_stdout',
'json_format': 'elasticsearch_json_format',
'json_fields': 'elasticsearch_json_fields'
}
}
# A mapping of old default values that we want to change and warn the user
# about. Mapping of section -> setting -> { old, replace, by_version }
deprecated_values = {
'core': {
'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
},
}
# This method transforms option names on every read, get, or set operation.
# This changes from the default behaviour of ConfigParser from lowercasing
# to instead be case-preserving
def optionxform(self, optionstr):
return optionstr
def __init__(self, default_config=None, *args, **kwargs):
super(AirflowConfigParser, self).__init__(*args, **kwargs)
self.airflow_defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.airflow_defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
self._validate_config_dependencies()
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
old, new, version = info
if self.get(section, name, fallback=None) == old:
# Make sure the env var option is removed, otherwise it
# would be read and used instead of the value we set
env_var = self._env_var_name(section, name)
os.environ.pop(env_var, None)
self.set(section, name, new)
warnings.warn(
'The {name} setting in [{section}] has the old default value '
'of {old!r}. This value has been changed to {new!r} in the '
'running config, but please update your config before Apache '
'Airflow {version}.'.format(
name=name, section=section, old=old, new=new, version=version
),
FutureWarning
)
self.is_validated = True
def _validate_config_dependencies(self):
"""
Validate that config values aren't invalid given other config values
or system-level limitations and requirements.
"""
if (
self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and
"sqlite" in self.get('core', 'sql_alchemy_conn')):
raise AirflowConfigException(
"error: cannot use sqlite with the {}".format(
self.get('core', 'executor')))
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
):
raise AirflowConfigException(
"error: owner_mode option should be either "
"'user' or 'ldapgroup' when filtering by owner is set")
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
self.get("webserver", "auth_backend") != (
'airflow.contrib.auth.backends.ldap_auth')
):
raise AirflowConfigException(
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
if self.has_option('core', 'mp_start_method'):
mp_start_method = self.get('core', 'mp_start_method')
start_method_options = multiprocessing.get_all_start_methods()
if mp_start_method not in start_method_options:
raise AirflowConfigException(
"mp_start_method should not be " + mp_start_method +
". Possible values are " + ", ".join(start_method_options))
@staticmethod
def _env_var_name(section, key):
return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
def _get_env_var_option(self, section, key):
# must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
env_var = self._env_var_name(section, key)
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
# alternatively AIRFLOW__{SECTION}__{KEY}_CMD (for a command)
env_var_cmd = env_var + '_CMD'
if env_var_cmd in os.environ:
# if this is a valid command key...
if (section, key) in self.sensitive_config_values:
return run_command(os.environ[env_var_cmd])
# alternatively AIRFLOW__{SECTION}__{KEY}_SECRET (to get from Secrets Backend)
env_var_secret_path = env_var + '_SECRET'
if env_var_secret_path in os.environ:
# if this is a valid secret path...
if (section, key) in self.sensitive_config_values:
return _get_config_value_from_secret_backend(os.environ[env_var_secret_path])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.sensitive_config_values:
if super(AirflowConfigParser, self).has_option(section, fallback_key):
command = super(AirflowConfigParser, self).get(section, fallback_key)
return run_command(command)
def _get_secret_option(self, section, key):
"""Get Config option values from Secret Backend"""
fallback_key = key + '_secret'
# if this is a valid secret key...
if (section, key) in self.sensitive_config_values:
if super(AirflowConfigParser, self).has_option(section, fallback_key):
secrets_path = super(AirflowConfigParser, self).get(section, fallback_key)
return _get_config_value_from_secret_backend(secrets_path)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
deprecated_name = self.deprecated_options.get(section, {}).get(key, None)
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
if deprecated_name:
option = self._get_env_var_option(section, deprecated_name)
if option is not None:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the config file
if super(AirflowConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(AirflowConfigParser, self).get(section, key, **kwargs))
if deprecated_name:
if super(AirflowConfigParser, self).has_option(section, deprecated_name):
self._warn_deprecate(section, key, deprecated_name)
return expand_env_var(super(AirflowConfigParser, self).get(
section,
deprecated_name,
**kwargs
))
# ...then commands
option = self._get_cmd_option(section, key)
if option:
return option
if deprecated_name:
option = self._get_cmd_option(section, deprecated_name)
if option:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then from secret backends
option = self._get_secret_option(section, key)
if option:
return option
if deprecated_name:
option = self._get_secret_option(section, deprecated_name)
if option:
self._warn_deprecate(section, key, deprecated_name)
return option
# ...then the default config
if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs:
return expand_env_var(
self.airflow_defaults.get(section, key, **kwargs))
else:
log.warning(
"section/key [%s/%s] not found in config", section, key
)
raise AirflowConfigException(
"section/key [{section}/{key}] not found "
"in config".format(section=section, key=key))
def getimport(self, section, key, **kwargs):
"""
Reads options, imports the full qualified name, and returns the object.
In case of failure, it throws an exception a clear message with the key aad the section names
:return: The object or None, if the option is empty
"""
full_qualified_path = conf.get(section=section, key=key, **kwargs)
if not full_qualified_path:
return None
try:
return import_string(full_qualified_path)
except ImportError as e:
log.error(e)
raise AirflowConfigException(
'The object could not be loaded. Please check "{key}" key in "{section}" section. '
'Current value: "{full_qualified_path}".'.format(
key=key, section=section, full_qualified_path=full_qualified_path)
)
def getboolean(self, section, key, **kwargs):
val = str(self.get(section, key, **kwargs)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val in ('t', 'true', '1'):
return True
elif val in ('f', 'false', '0'):
return False
else:
raise ValueError(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key, **kwargs):
return int(self.get(section, key, **kwargs))
def getfloat(self, section, key, **kwargs):
return float(self.get(section, key, **kwargs))
def read(self, filenames, **kwargs):
super(AirflowConfigParser, self).read(filenames, **kwargs)
self._validate()
def read_dict(self, *args, **kwargs):
super(AirflowConfigParser, self).read_dict(*args, **kwargs)
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
# UNSET to avoid logging a warning about missing values
self.get(section, option, fallback=_UNSET)
return True
except (NoOptionError, NoSectionError):
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(AirflowConfigParser, self).has_option(section, option):
super(AirflowConfigParser, self).remove_option(section, option)
if self.airflow_defaults.has_option(section, option) and remove_default:
self.airflow_defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:rtype: dict
"""
if (section not in self._sections and
section not in self.airflow_defaults._sections):
return None
_section = copy.deepcopy(self.airflow_defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
for env_var in sorted(os.environ.keys()):
if env_var.startswith(section_prefix):
key = env_var.replace(section_prefix, '')
if key.endswith("_CMD"):
key = key[:-4]
key = key.lower()
_section[key] = self._get_env_var_option(section, key)
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def write(self, fp, space_around_delimiters=True):
# This is based on the configparser.RawConfigParser.write method code to add support for
# reading options from environment variables.
if space_around_delimiters:
d = " {} ".format(self._delimiters[0]) # type: ignore
else:
d = self._delimiters[0] # type: ignore
if self._defaults:
self._write_section(fp, self.default_section, self._defaults.items(), d) # type: ignore
for section in self._sections:
self._write_section(fp, section, self.getsection(section).items(), d) # type: ignore
def as_dict(
self, display_source=False, display_sensitive=False, raw=False,
include_env=True, include_cmds=True, include_secret=True
):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg', 'default', 'env var', or 'cmd'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
:param raw: Should the values be output as interpolated values, or the
"raw" form that can be fed back in to ConfigParser
:type raw: bool
:param include_env: Should the value of configuration from AIRFLOW__
environment variables be included or not
:type include_env: bool
:param include_cmds: Should the result of calling any *_cmd config be
set (True, default), or should the _cmd options be left as the
command to run (False)
:type include_cmds: bool
:param include_secret: Should the result of calling any *_secret config be
set (True, default), or should the _secret options be left as the
path to get the secret from (False)
:type include_secret: bool
:return: Dictionary, where the key is the name of the section and the content is
the dictionary with the name of the parameter and its value.
"""
cfg = {}
configs = [
('default', self.airflow_defaults),
('airflow.cfg', self),
]
for (source_name, config) in configs:
for section in config.sections():
sect = cfg.setdefault(section, OrderedDict())
for (k, val) in config.items(section=section, raw=raw):
if display_source:
val = (val, source_name)
sect[k] = val
# add env vars and overwrite because they have priority
if include_env:
for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
try:
_, section, key = ev.split('__', 2)
opt = self._get_env_var_option(section, key)
except ValueError:
continue
if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE':
opt = '< hidden >'
elif raw:
opt = opt.replace('%', '%%')
if display_source:
opt = (opt, 'env var')
section = section.lower()
# if we lower key for kubernetes_environment_variables section,
# then we won't be able to set any Airflow environment
# variables. Airflow only parse environment variables starts
# with AIRFLOW_. Therefore, we need to make it a special case.
if section != 'kubernetes_environment_variables':
key = key.lower()
cfg.setdefault(section, OrderedDict()).update({key: opt})
# add bash commands
if include_cmds:
for (section, key) in self.sensitive_config_values:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'cmd')
elif raw:
opt = opt.replace('%', '%%')
cfg.setdefault(section, OrderedDict()).update({key: opt})
del cfg[section][key + '_cmd']
# add config from secret backends
if include_secret:
for (section, key) in self.sensitive_config_values:
opt = self._get_secret_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'secret')
elif raw:
opt = opt.replace('%', '%%')
cfg.setdefault(section, OrderedDict()).update({key: opt})
del cfg[section][key + '_secret']
return cfg
def load_test_config(self):
"""
Load the unit test configuration.
Note: this is not reversible.
"""
# override any custom settings with defaults
log.info("Overriding settings with defaults from %s", DEFAULT_CONFIG_FILE_PATH)
self.read_string(parameterized_config(DEFAULT_CONFIG))
# then read test config
log.info("Reading default test configuration from %s", TEST_CONFIG_FILE_PATH)
self.read_string(parameterized_config(TEST_CONFIG))
# then read any "custom" test settings
log.info("Reading test configuration from %s", TEST_CONFIG_FILE)
self.read(TEST_CONFIG_FILE)
def _warn_deprecate(self, section, key, deprecated_name):
warnings.warn(
'The {old} option in [{section}] has been renamed to {new} - the old '
'setting has been used, but please update your config.'.format(
old=deprecated_name,
new=key,
section=section,
),
DeprecationWarning,
stacklevel=3,
)
class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
as_command_stdout = {('core', 'sql_alchemy_conn'), ('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'result_backend')}
def __init__(self, default_config=None, *args, **kwargs):
super(AirflowConfigParser, self).__init__(*args, **kwargs)
self.defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
if (self.get("core", "executor") != 'SequentialExecutor'
and "sqlite" in self.get('core', 'sql_alchemy_conn')):
raise AirflowConfigException(
"error: cannot use sqlite with the {}".format(
self.get('core', 'executor')))
elif (self.getboolean("webserver", "authenticate") and self.get(
"webserver", "owner_mode") not in ['user', 'ldapgroup']):
raise AirflowConfigException(
"error: owner_mode option should be either "
"'user' or 'ldapgroup' when filtering by owner is set")
elif (self.getboolean("webserver", "authenticate")
and self.get("webserver", "owner_mode").lower() == 'ldapgroup'
and self.get("webserver", "auth_backend") !=
('airflow.contrib.auth.backends.ldap_auth')):
raise AirflowConfigException(
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
self.is_validated = True
def _get_env_var_option(self, section, key):
# must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.as_command_stdout:
if super(AirflowConfigParser, self) \
.has_option(section, fallback_key):
command = super(AirflowConfigParser, self) \
.get(section, fallback_key)
return run_command(command)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
# ...then the config file
if super(AirflowConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(AirflowConfigParser, self).get(section, key, **kwargs))
# ...then commands
option = self._get_cmd_option(section, key)
if option:
return option
# ...then the default config
if self.defaults.has_option(section, key):
return expand_env_var(self.defaults.get(section, key, **kwargs))
else:
log.warning(
"section/key [{section}/{key}] not found in config".format(
**locals()))
raise AirflowConfigException(
"section/key [{section}/{key}] not found "
"in config".format(**locals()))
def getboolean(self, section, key):
val = str(self.get(section, key)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val.lower() in ('t', 'true', '1'):
return True
elif val.lower() in ('f', 'false', '0'):
return False
else:
raise AirflowConfigException(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key):
return int(self.get(section, key))
def getfloat(self, section, key):
return float(self.get(section, key))
def read(self, filenames):
super(AirflowConfigParser, self).read(filenames)
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
self.get(section, option)
return True
except AirflowConfigException:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(AirflowConfigParser, self).has_option(section, option):
super(AirflowConfigParser, self).remove_option(section, option)
if self.defaults.has_option(section, option) and remove_default:
self.defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:return: dict
"""
if section not in self._sections and section not in self.defaults._sections:
return None
_section = copy.deepcopy(self.defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(self, display_source=False, display_sensitive=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg' or 'default'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
"""
cfg = copy.deepcopy(self.defaults._sections)
cfg.update(copy.deepcopy(self._sections))
# remove __name__ (affects Python 2 only)
for options in cfg.values():
options.pop('__name__', None)
# add source
if display_source:
for section in cfg:
for k, v in cfg[section].items():
cfg[section][k] = (v, 'airflow config')
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
opt = None
if opt:
if (not display_sensitive
and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
opt = '< hidden >'
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(),
OrderedDict()).update({key.lower(): opt})
# add bash commands
for (section, key) in self.as_command_stdout:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'bash cmd')
cfg.setdefault(section, OrderedDict()).update({key: opt})
return cfg
def load_test_config(self):
"""
Load the unit test configuration.
Note: this is not reversible.
"""
# override any custom settings with defaults
self.defaults.read_string(parameterized_config(DEFAULT_CONFIG))
# then read test config
self.read_string(parameterized_config(TEST_CONFIG))
# then read any "custom" test settings
self.read(TEST_CONFIG_FILE)
class SpacetimeGISConfigParser(ConfigParser):
def __init__(self, default_config=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.spacetimegis_defaults = ConfigParser(*args, **kwargs)
if not default_config:
self.spacetimegis_defaults.read_string(default_config)
@staticmethod
def _env_var_name(section, key):
return 'SPACETIMEGIS__{S}__{K}'.format(S=section.upper(),
K=key.upper())
def _get_env_var_option(self, section, key):
# must have format SPACETIMEGIS__{SECTION}__{KEY} (note double underscore)
env_var = self._env_var_name(section, key)
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
# ...then the config file
if super().has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(super().get(section, key, **kwargs))
# ...then the default config
if self.spacetimegis_defaults.has_option(section,
key) or 'fallback' in kwargs:
return expand_env_var(
self.spacetimegis_defaults.get(section, key, **kwargs))
def getboolean(self, section, key, **kwargs):
val = str(self.get(section, key, **kwargs)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val in ('t', 'true', '1'):
return True
elif val in ('f', 'false', '0'):
return False
else:
# logger.writelog(LogLevel.error,
# 'The value for configuration option "{}:{}" is not a '
# 'boolean (received "{}").'.format(section, key, val))
raise ValueError(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key, **kwargs):
return int(self.get(section, key, **kwargs))
def getfloat(self, section, key, **kwargs):
return float(self.get(section, key, **kwargs))
def read(self, filenames, **kwargs):
super().read(filenames, **kwargs)
def read_dict(self, *args, **kwargs):
super().read_dict(*args, **kwargs)
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
# UNSET to avoid logging a warning about missing values
self.get(section, option, fallback=_UNSET)
return True
except NoOptionError:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super().has_option(section, option):
super().remove_option(section, option)
if self.spacetimegis_defaults.has_option(section,
option) and remove_default:
self.spacetimegis_defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:rtype: dict
"""
if (section not in self._sections
and section not in self.spacetimegis_defaults._sections):
return None
_section = copy.deepcopy(self.spacetimegis_defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
section_prefix = 'SPACETIMEGIS__{S}__'.format(S=section.upper())
for env_var in sorted(os.environ.keys()):
if env_var.startswith(section_prefix):
key = env_var.replace(section_prefix, '').lower()
_section[key] = self._get_env_var_option(section, key)
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(self,
display_source=False,
display_sensitive=False,
raw=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg', 'default', 'env var', or 'cmd'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
:param raw: Should the values be output as interpolated values, or the
"raw" form that can be fed back in to ConfigParser
:type raw: bool
"""
cfg = {}
configs = [
('default', self.spacetimegis_defaults),
('spacetimegis.cfg', self),
]
for (source_name, config) in configs:
for section in config.sections():
sect = cfg.setdefault(section, OrderedDict())
for (k, val) in config.items(section=section, raw=raw):
if display_source:
val = (val, source_name)
sect[k.upper()] = val
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('SPACETIMEGIS__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
continue
if not display_sensitive and ev != 'SPACETIMEGIS__CORE__UNIT_TEST_MODE':
opt = '< hidden >'
elif raw:
opt = opt.replace('%', '%%')
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(),
OrderedDict()).update({key.lower(): opt})
return cfg
def as_all_dict(self,
display_source=False,
display_sensitive=False,
raw=False):
tmp = self.as_dict(display_source, display_sensitive, raw)
all_cfg = {}
for val in tmp.values():
if not all_cfg:
all_cfg = val.copy()
else:
all_cfg.update(val)
return all_cfg
class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
as_command_stdout = {
('core', 'sql_alchemy_conn'),
('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'result_backend')
}
def __init__(self, default_config=None, *args, **kwargs):
super(AirflowConfigParser, self).__init__(*args, **kwargs)
self.defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
self.defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
if (
self.get("core", "executor") != 'SequentialExecutor' and
"sqlite" in self.get('core', 'sql_alchemy_conn')):
raise AirflowConfigException(
"error: cannot use sqlite with the {}".format(
self.get('core', 'executor')))
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
):
raise AirflowConfigException(
"error: owner_mode option should be either "
"'user' or 'ldapgroup' when filtering by owner is set")
elif (
self.getboolean("webserver", "authenticate") and
self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
self.get("webserver", "auth_backend") != (
'airflow.contrib.auth.backends.ldap_auth')
):
raise AirflowConfigException(
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
self.is_validated = True
def _get_env_var_option(self, section, key):
# must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.as_command_stdout:
if super(AirflowConfigParser, self) \
.has_option(section, fallback_key):
command = super(AirflowConfigParser, self) \
.get(section, fallback_key)
return run_command(command)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
return option
# ...then the config file
if super(AirflowConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(AirflowConfigParser, self).get(section, key, **kwargs))
# ...then commands
option = self._get_cmd_option(section, key)
if option:
return option
# ...then the default config
if self.defaults.has_option(section, key):
return expand_env_var(
self.defaults.get(section, key, **kwargs))
else:
log.warning(
"section/key [{section}/{key}] not found in config".format(**locals())
)
raise AirflowConfigException(
"section/key [{section}/{key}] not found "
"in config".format(**locals()))
def getboolean(self, section, key):
val = str(self.get(section, key)).lower().strip()
if '#' in val:
val = val.split('#')[0].strip()
if val.lower() in ('t', 'true', '1'):
return True
elif val.lower() in ('f', 'false', '0'):
return False
else:
raise AirflowConfigException(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key):
return int(self.get(section, key))
def getfloat(self, section, key):
return float(self.get(section, key))
def read(self, filenames):
super(AirflowConfigParser, self).read(filenames)
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
self.get(section, option)
return True
except AirflowConfigException:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(AirflowConfigParser, self).has_option(section, option):
super(AirflowConfigParser, self).remove_option(section, option)
if self.defaults.has_option(section, option) and remove_default:
self.defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:return: dict
"""
if section not in self._sections and section not in self.defaults._sections:
return None
_section = copy.deepcopy(self.defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(self, display_source=False, display_sensitive=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'airflow.cfg' or 'default'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
"""
cfg = copy.deepcopy(self.defaults._sections)
cfg.update(copy.deepcopy(self._sections))
# remove __name__ (affects Python 2 only)
for options in cfg.values():
options.pop('__name__', None)
# add source
if display_source:
for section in cfg:
for k, v in cfg[section].items():
cfg[section][k] = (v, 'airflow config')
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
opt = None
if opt:
if (
not display_sensitive
and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
opt = '< hidden >'
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(), OrderedDict()).update(
{key.lower(): opt})
# add bash commands
for (section, key) in self.as_command_stdout:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'bash cmd')
cfg.setdefault(section, OrderedDict()).update({key: opt})
return cfg
def load_test_config(self):
"""
Load the unit test configuration.
Note: this is not reversible.
"""
# override any custom settings with defaults
self.defaults.read_string(parameterized_config(DEFAULT_CONFIG))
# then read test config
self.read_string(parameterized_config(TEST_CONFIG))
# then read any "custom" test settings
self.read(TEST_CONFIG_FILE)
class XToolConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
as_command_stdout = {
('core', 'sql_alchemy_conn'),
('core', 'fernet_key'),
('celery', 'broker_url'),
('celery', 'result_backend'),
}
def __init__(self, default_config=None, *args, **kwargs):
super(XToolConfigParser, self).__init__(*args, **kwargs)
self.defaults = ConfigParser(*args, **kwargs)
# 读取配置字符串
if default_config is not None:
self.defaults.read_string(default_config)
self.is_validated = False
def _validate(self):
self.is_validated = True
def _get_env_var_option(self, section, key):
"""把环境变量的值中包含的”~”和”~user”转换成用户目录，并获取配置结果值 ."""
# must have format XTOOL__{SECTION}__{KEY} (note double underscore)
env_var = 'XTOOL__{S}__{K}'.format(S=section.upper(), K=key.upper())
if env_var in os.environ:
return expand_env_var(os.environ[env_var])
def _get_cmd_option(self, section, key):
"""从配置项中获取指令，并执行指令获取指令执行后的返回值
- 如果key不存在_cmd结尾，则获取key的值
- 如果key没有配置 且 key以_cmd结尾，则获取key的值，并执行值表示的表达式，返回表达式的结果
"""
fallback_key = key + '_cmd'
# if this is a valid command key...
if (section, key) in self.as_command_stdout:
if self.has_option(section, fallback_key):
command = self.get(section, fallback_key)
return run_command(command)
def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()
# 首先从环境变量中获取配置值，如果环境变量中存在，则不再从配置文件中获取
option = self._get_env_var_option(section, key)
if option is not None:
return option
# 然后从配置文件中获取
if super(XToolConfigParser, self).has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
# separate the config from default config.
return expand_env_var(
super(XToolConfigParser, self).get(section, key, **kwargs))
# 执行表达式，获取结果
option = self._get_cmd_option(section, key)
if option:
return option
# ...then the default config
if self.defaults.has_option(section, key):
return expand_env_var(
self.defaults.get(section, key, **kwargs))
else:
log.warning(
"section/key [{section}/{key}] not found in config".format(**locals())
)
# 配置不存在，抛出异常
raise XToolConfigException(
"section/key [{section}/{key}] not found "
"in config".format(**locals()))
def getboolean(self, section, key):
val = str(self.get(section, key)).lower().strip()
# 去掉结尾的注释
if '#' in val:
val = val.split('#')[0].strip()
if val.lower() in ('t', 'true', '1'):
return True
elif val.lower() in ('f', 'false', '0'):
return False
else:
raise XToolConfigException(
'The value for configuration option "{}:{}" is not a '
'boolean (received "{}").'.format(section, key, val))
def getint(self, section, key):
return int(self.get(section, key))
def getfloat(self, section, key):
return float(self.get(section, key))
def read(self, filenames):
"""读取多个配置文件，并进行校验
@note: 执行此函数，获取的配置项会覆盖掉构造函数中的默认配置 default_config
"""
super(XToolConfigParser, self).read(filenames, encoding="utf-8")
self._validate()
def has_option(self, section, option):
try:
# Using self.get() to avoid reimplementing the priority order
# of config variables (env, config, cmd, defaults)
self.get(section, option)
return True
except XToolConfigException:
return False
def remove_option(self, section, option, remove_default=True):
"""
Remove an option if it exists in config from a file or
default config. If both of config have the same option, this removes
the option in both configs unless remove_default=False.
"""
if super(XToolConfigParser, self).has_option(section, option):
super(XToolConfigParser, self).remove_option(section, option)
if self.defaults.has_option(section, option) and remove_default:
self.defaults.remove_option(section, option)
def getsection(self, section):
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
:param section: section from the config
:return: dict
"""
if section not in self._sections and section not in self.defaults._sections:
return None
_section = copy.deepcopy(self.defaults._sections[section])
if section in self._sections:
_section.update(copy.deepcopy(self._sections[section]))
# 遍历section下所有的key，对value进行格式化处理
for key, val in iteritems(_section):
try:
val = int(val)
except ValueError:
try:
val = float(val)
except ValueError:
if val.lower() in ('t', 'true'):
val = True
elif val.lower() in ('f', 'false'):
val = False
_section[key] = val
return _section
def as_dict(self, display_source=False, display_sensitive=False):
"""
Returns the current configuration as an OrderedDict of OrderedDicts.
:param display_source: If False, the option value is returned. If True,
a tuple of (option_value, source) is returned. Source is either
'xTool.cfg' or 'default'.
:type display_source: bool
:param display_sensitive: If True, the values of options set by env
vars and bash commands will be displayed. If False, those options
are shown as '< hidden >'
:type display_sensitive: bool
"""
cfg = copy.deepcopy(self.defaults._sections)
cfg.update(copy.deepcopy(self._sections))
# remove __name__ (affects Python 2 only)
for options in cfg.values():
options.pop('__name__', None)
# add source
if display_source:
for section in cfg:
for k, v in cfg[section].items():
cfg[section][k] = (v, 'xTool config')
# add env vars and overwrite because they have priority
for ev in [ev for ev in os.environ if ev.startswith('XTOOL__')]:
try:
_, section, key = ev.split('__')
opt = self._get_env_var_option(section, key)
except ValueError:
opt = None
if opt:
if (
not display_sensitive and
ev != 'XTOOL__CORE__UNIT_TEST_MODE'):
opt = '< hidden >'
if display_source:
opt = (opt, 'env var')
cfg.setdefault(section.lower(), OrderedDict()).update(
{key.lower(): opt})
# add bash commands
for (section, key) in self.as_command_stdout:
opt = self._get_cmd_option(section, key)
if opt:
if not display_sensitive:
opt = '< hidden >'
if display_source:
opt = (opt, 'bash cmd')
cfg.setdefault(section, OrderedDict()).update({key: opt})
return cfg