def _set_data_to_obj(
self,
data,
identifier,
source_file,
file_merge=None,
key=False,
env=False,
):
"""Calls setttings.set to add the keys"""
# data 1st level keys should be transformed to upper case.
data = {upperfy(k): v for k, v in data.items()}
if key:
key = upperfy(key)
# is there a `dynaconf_merge` inside an `[env]`?
file_merge = file_merge or data.pop("DYNACONF_MERGE", False)
if not key:
self.obj.update(
data,
loader_identifier=identifier,
merge=file_merge,
)
elif key in data:
self.obj.set(
key,
data.get(key),
loader_identifier=identifier,
merge=file_merge,
)
def __call__(self, data):
"""Filter incoming data by prefix"""
len_prefix = len(self.prefix)
return {
upperfy(key[len_prefix:]): value
for key, value in data.items()
if upperfy(key[:len_prefix]) == self.prefix
}
def load_from_env(
obj,
prefix=False,
key=None,
silent=False,
identifier=IDENTIFIER,
env=False, # backwards compatibility bc renamed param
):
if prefix is False and env is not False:
prefix = env
env_ = ""
if prefix is not False:
if not isinstance(prefix, str):
raise TypeError("`prefix/env` must be str or False")
prefix = prefix.upper()
env_ = f"{prefix}_"
if key:
key = upperfy(key)
value = environ.get(f"{env_}{key}")
if value:
try: # obj is a Settings
obj.set(key, value, loader_identifier=identifier, tomlfy=True)
except AttributeError: # obj is a dict
obj[key] = parse_conf_data(value, tomlfy=True)
else:
trim_len = len(env_)
data = {
key[trim_len:]: parse_conf_data(data, tomlfy=True)
for key, data in environ.items() if key.startswith(env_)
}
if data:
obj.update(data, loader_identifier=identifier)
def write(obj, data=None, **kwargs):
"""Write a value in to loader source
:param obj: settings object
:param data: vars to be stored
:param kwargs: vars to be stored
:return:
"""
if obj.REDIS_ENABLED_FOR_DYNACONF is False:
raise RuntimeError(
"Redis is not configured \n"
"export REDIS_ENABLED_FOR_DYNACONF=true\n"
"and configure the REDIS_FOR_DYNACONF_* variables"
)
client = StrictRedis(**obj.REDIS_FOR_DYNACONF)
holder = obj.get("ENVVAR_PREFIX_FOR_DYNACONF").upper()
# add env to holder
holder = f"{holder}_{obj.current_env.upper()}"
data = data or {}
data.update(kwargs)
if not data:
raise AttributeError("Data must be provided")
redis_data = {
upperfy(key): unparse_conf_data(value) for key, value in data.items()
}
client.hmset(holder.upper(), redis_data)
load(obj)
def split_vars(_vars):
"""Splits values like foo=bar=zaz in {'foo': 'bar=zaz'}"""
return ({
upperfy(k.strip()): parse_conf_data(
v.strip(), tomlfy=True, box_settings=settings)
for k, _, v in [item.partition("=") for item in _vars]
} if _vars else {})
def get(
self,
key,
default=None,
cast=None,
fresh=False,
dotted_lookup=True,
parent=None,
):
"""
Get a value from settings store, this is the prefered way to access::
>>> from dynaconf import settings
>>> settings.get('KEY')
:param key: The name of the setting value, will always be upper case
:param default: In case of not found it will be returned
:param cast: Should cast in to @int, @float, @bool or @json ?
:param fresh: Should reload from loaders store before access?
:param dotted_lookup: Should perform dotted-path lookup?
:param parent: Is there a pre-loaded parent in a nested data?
:return: The value if found, default or None
"""
nested_sep = self._store.get("NESTED_SEPARATOR_FOR_DYNACONF")
if nested_sep and nested_sep in key:
# turn FOO__bar__ZAZ in `FOO.bar.ZAZ`
key = key.replace(nested_sep, ".")
if "." in key and dotted_lookup:
return self._dotted_get(
dotted_key=key,
default=default,
cast=cast,
fresh=fresh,
parent=parent,
)
if default is not None:
# default values should behave exactly Dynaconf parsed values
if isinstance(default, list):
default = BoxList(default)
elif isinstance(default, dict):
default = DynaBox(default)
key = upperfy(key)
if key in self._deleted:
return default
if (
fresh
or self._fresh
or key in getattr(self, "FRESH_VARS_FOR_DYNACONF", ())
) and key not in UPPER_DEFAULT_SETTINGS:
self.unset(key)
self.execute_loaders(key=key)
data = (parent or self.store).get(key, default)
if cast:
data = get_converter(cast, data, box_settings=self)
return data
def get(self, item, default=None, *args, **kwargs):
if item not in self: # toggle case
item = item.lower() if item.isupper() else upperfy(item)
value = super(DynaBox, self).get(item, empty, *args, **kwargs)
if value is empty:
# see Issue: #486
return self._case_insensitive_get(item, default)
return value
def get(key, default=None):
value = os.environ.get(upperfy(key))
# compatibility with renamed variables
for old, new in RENAMED_VARS.items():
value = try_renamed(key, value, old, new)
return (parse_conf_data(value, tomlfy=True)
if value is not None else default)
def load_from_env(
obj,
prefix=False,
key=None,
silent=False,
identifier=IDENTIFIER,
env=False, # backwards compatibility bc renamed param
):
if prefix is False and env is not False:
prefix = env
env_ = ""
if prefix is not False:
if not isinstance(prefix, str):
raise TypeError("`prefix/env` must be str or False")
prefix = prefix.upper()
env_ = f"{prefix}_"
# Load a single environment variable explicitly.
if key:
key = upperfy(key)
value = environ.get(f"{env_}{key}")
if value:
try: # obj is a Settings
obj.set(key, value, loader_identifier=identifier, tomlfy=True)
except AttributeError: # obj is a dict
obj[key] = parse_conf_data(
value, tomlfy=True, box_settings=obj
)
# Load environment variables in bulk (when matching).
else:
# Only known variables should be loaded from environment?
ignore_unknown = obj.get("IGNORE_UNKNOWN_ENVVARS_FOR_DYNACONF")
trim_len = len(env_)
data = {
key[trim_len:]: parse_conf_data(
data, tomlfy=True, box_settings=obj
)
for key, data in environ.items()
if key.startswith(env_)
and not (
# Ignore environment variables that haven't been
# pre-defined in settings space.
ignore_unknown
and obj.get(key[trim_len:], default=missing) is missing
)
}
# Update the settings space based on gathered data from environment.
if data:
filter_strategy = obj.get("FILTER_STRATEGY")
if filter_strategy:
data = filter_strategy(data)
obj.update(data, loader_identifier=identifier)
def _set_data_to_obj(
self,
data,
identifier,
file_merge=None,
key=False,
file_dotted_lookup=None,
):
"""Calls settings.set to add the keys"""
# data 1st level keys should be transformed to upper case.
data = {upperfy(k): v for k, v in data.items()}
if key:
key = upperfy(key)
if self.obj.filter_strategy:
data = self.obj.filter_strategy(data)
# is there a `dynaconf_merge` inside an `[env]`?
file_merge = file_merge or data.pop("DYNACONF_MERGE", False)
# If not passed or passed as None,
# look for inner [env] value, or default settings.
if file_dotted_lookup is None:
file_dotted_lookup = data.pop(
"DYNACONF_DOTTED_LOOKUP",
self.obj.get("DOTTED_LOOKUP_FOR_DYNACONF"),
)
if not key:
self.obj.update(
data,
loader_identifier=identifier,
merge=file_merge,
dotted_lookup=file_dotted_lookup,
)
elif key in data:
self.obj.set(
key,
data.get(key),
loader_identifier=identifier,
merge=file_merge,
dotted_lookup=file_dotted_lookup,
)
def unset(self, key, force=False):
"""Unset on all references
:param key: The key to be unset
:param force: Bypass default checks and force unset
"""
key = upperfy(key.strip())
if (key not in dir(default_settings) and key not in self._defaults
or force):
delattr(self, key)
self.store.pop(key, None)
def exists(self, key, fresh=False):
"""Check if key exists
:param key: the name of setting variable
:param fresh: if key should be taken from source direclty
:return: Boolean
"""
key = upperfy(key)
if key in self._deleted:
return False
return self.get(key, fresh=fresh, default=missing) is not missing
def test_upperfy():
assert upperfy("foo") == "FOO"
assert upperfy("foo__bar") == "FOO__bar"
assert upperfy("foo__bar__ZAZ") == "FOO__bar__ZAZ"
assert (upperfy("foo__bar__ZAZ__naz__TAZ_ZAZ") ==
"FOO__bar__ZAZ__naz__TAZ_ZAZ")
assert upperfy("foo_bar") == "FOO_BAR"
assert upperfy("foo_BAR") == "FOO_BAR"
def unset(self, key, force=False):
"""Unset on all references
:param key: The key to be unset
:param force: Bypass default checks and force unset
"""
key = upperfy(key.strip())
if (key not in dir(default_settings) and key not in self._defaults
or force):
with suppress(KeyError, AttributeError):
# AttributeError can happen when a LazyValue consumes
# a previously deleted key
delattr(self, key)
del self.store[key]
def get(
self,
key,
default=None,
cast=None,
fresh=False,
dotted_lookup=True,
parent=None,
):
"""
Get a value from settings store, this is the prefered way to access::
>>> from dynaconf import settings
>>> settings.get('KEY')
:param key: The name of the setting value, will always be upper case
:param default: In case of not found it will be returned
:param cast: Should cast in to @int, @float, @bool or @json ?
:param fresh: Should reload from loaders store before access?
:param dotted_lookup: Should perform dotted-path lookup?
:param parent: Is there a pre-loaded parent in a nested data?
:return: The value if found, default or None
"""
if "." in key and dotted_lookup:
return self._dotted_get(
dotted_key=key,
default=default,
cast=cast,
fresh=fresh,
parent=parent,
)
key = upperfy(key)
if key in self._deleted:
return default
if (
fresh
or self._fresh
or key in getattr(self, "FRESH_VARS_FOR_DYNACONF", ())
) and key not in dir(default_settings):
self.unset(key)
self.execute_loaders(key=key)
data = (parent or self.store).get(key, default)
if cast:
data = converters.get(cast)(data)
return data
def populate_obj(self, obj, keys=None, ignore=None):
"""Given the `obj` populate it using self.store items.
:param obj: An object to be populated, a class instance.
:param keys: A list of keys to be included.
:param ignore: A list of keys to be excluded.
"""
keys = keys or self.keys()
for key in keys:
key = upperfy(key)
if ignore and key in ignore:
continue
value = self.get(key, empty)
if value is not empty:
setattr(obj, key, value)
def get_environ(self, key, default=None, cast=None):
"""Get value from environment variable using os.environ.get
:param key: The name of the setting value, will always be upper case
:param default: In case of not found it will be returned
:param cast: Should cast in to @int, @float, @bool or @json ?
or cast must be true to use cast inference
:return: The value if found, default or None
"""
key = upperfy(key)
data = self.environ.get(key, default)
if data:
if cast in converters:
data = converters.get(cast)(data)
if cast is True:
data = parse_conf_data(data, tomlfy=True)
return data
def delete(obj, key=None):
"""
Delete a single key if specified, or all env if key is none
:param obj: settings object
:param key: key to delete from store location
:return: None
"""
client = StrictRedis(**obj.REDIS_FOR_DYNACONF)
holder = obj.get("ENVVAR_PREFIX_FOR_DYNACONF").upper()
# add env to holder
holder = f"{holder}_{obj.current_env.upper()}"
if key:
client.hdel(holder.upper(), upperfy(key))
obj.unset(key)
else:
keys = client.hkeys(holder.upper())
client.delete(holder.upper())
obj.unset_all(keys)
def write(settings_path, settings_data, merge=True):
"""Write data to a settings file.
:param settings_path: the filepath
:param settings_data: a dictionary with data
:param merge: boolean if existing file should be merged with new data
"""
settings_path = Path(settings_path)
if settings_path.exists() and merge: # pragma: no cover
existing = DynaconfDict()
load(existing, str(settings_path))
object_merge(existing, settings_data)
with io.open(
str(settings_path),
"w",
encoding=default_settings.ENCODING_FOR_DYNACONF,
) as f:
f.writelines([
"{} = {}\n".format(upperfy(k), repr(v))
for k, v in settings_data.items()
])
def set(
self,
key,
value,
loader_identifier=None,
tomlfy=False,
dotted_lookup=True,
is_secret=False,
merge=False,
):
"""Set a value storing references for the loader
:param key: The key to store
:param value: The value to store
:param loader_identifier: Optional loader name e.g: toml, yaml etc.
:param tomlfy: Bool define if value is parsed by toml (defaults False)
:param is_secret: Bool define if secret values is hidden on logs.
:param merge: Bool define if existing nested data will be merged.
"""
nested_sep = self.get("NESTED_SEPARATOR_FOR_DYNACONF")
if nested_sep and nested_sep in key:
# turn FOO__bar__ZAZ in `FOO.bar.ZAZ`
key = key.replace(nested_sep, ".")
if "." in key and dotted_lookup is True:
return self._dotted_set(key,
value,
loader_identifier=loader_identifier,
tomlfy=tomlfy)
value = parse_conf_data(value, tomlfy=tomlfy)
key = upperfy(key.strip())
existing = getattr(self, key, None)
if getattr(value, "_dynaconf_del", None):
# just in case someone use a `@del` in a first level var.
self.unset(key, force=True)
return
if getattr(value, "_dynaconf_reset", False): # pragma: no cover
# just in case someone use a `@reset` in a first level var.
# NOTE: @reset/Reset is deprecated in v3.0.0
value = value.unwrap()
if getattr(value, "_dynaconf_merge", False):
# just in case someone use a `@merge` in a first level var
if existing:
object_merge(existing, value.unwrap())
value = value.unwrap()
if existing is not None and existing != value:
# `dynaconf_merge` used in file root `merge=True`
if merge:
object_merge(existing, value)
else:
# `dynaconf_merge` may be used within the key structure
value = self._merge_before_set(key, existing, value, is_secret)
if isinstance(value, dict):
value = DynaBox(value)
setattr(self, key, value)
self.store[key] = value
self._deleted.discard(key)
# set loader identifiers so cleaners know which keys to clean
if loader_identifier and loader_identifier in self.loaded_by_loaders:
self.loaded_by_loaders[loader_identifier][key] = value
elif loader_identifier:
self.loaded_by_loaders[loader_identifier] = {key: value}
elif loader_identifier is None:
# if .set is called without loader identifier it becomes
# a default value and goes away only when explicitly unset
self._defaults[key] = value
def exists_in_environ(self, key):
"""Return True if env variable is exported"""
return upperfy(key) in self.environ
def __init__(self, prefix):
if not isinstance(prefix, str):
raise TypeError("`SETTINGS_FILE_PREFIX` must be str")
self.prefix = "{}_".format(upperfy(prefix))
def _list(env, key, more, loader, _all=False, output=None, flat=False):
"""Lists all user defined config values
and if `--all` is passed it also shows dynaconf internal variables.
"""
if env:
env = env.strip()
if key:
key = key.strip()
if loader:
loader = loader.strip()
if env:
settings.setenv(env)
cur_env = settings.current_env.lower()
click.echo(
click.style(
"Working in %s environment " % cur_env,
bold=True,
bg="bright_blue",
fg="bright_white",
))
if not loader:
data = settings.as_dict(env=env, internal=_all)
else:
identifier = "{}_{}".format(loader, cur_env)
data = settings._loaded_by_loaders.get(identifier, {})
data = data or settings._loaded_by_loaders.get(loader, {})
# remove to avoid displaying twice
data.pop("SETTINGS_MODULE", None)
def color(_k):
if _k in dir(default_settings):
return "blue"
return "green"
def format_setting(_k, _v):
return "{key}{data_type} {value}".format(
key=click.style(_k, bg=color(_k), fg="white"),
data_type=click.style("<{}>".format(type(_v).__name__),
bg="bright_black",
fg="white"),
value=pprint.pformat(_v),
)
if not key:
datalines = "\n".join(
format_setting(k, v) for k, v in data.items()
if k not in data.get("RENAMED_VARS", []))
(click.echo_via_pager if more else click.echo)(datalines)
if output:
loaders.write(output, data, env=not flat and cur_env)
else:
key = upperfy(key)
value = data.get(key)
if not value:
click.echo(click.style("Key not found", bg="red", fg="white"))
return
click.echo(format_setting(key, value))
if output:
loaders.write(output, {key: value}, env=not flat and cur_env)
if env:
settings.setenv()
def _read(self, files, envs, silent=True, key=None):
for source_file in files:
if source_file.endswith(self.extensions):
try:
with io.open(
source_file,
encoding=self.obj.get("ENCODING_FOR_DYNACONF",
"utf-8"),
) as open_file:
source_data = self.file_reader(open_file)
self.obj.logger.debug("{}_loader: {}".format(
self.identifier, source_file))
except IOError:
self.obj.logger.debug(
"{}_loader: {} (Ignored, file not Found)".format(
self.identifier, source_file))
source_data = None
else:
# for tests it is possible to pass string
source_data = self.string_reader(source_file)
if not source_data:
continue
# env name is checked in lower
source_data = {
k.lower(): value
for k, value in source_data.items()
}
# is there a `dynaconf_merge` on top level of file?
file_merge = source_data.get("dynaconf_merge")
# all lower case for comparison
base_envs = [
# DYNACONF or MYPROGRAM
(self.obj.get("ENVVAR_PREFIX_FOR_DYNACONF") or "").lower(),
# DEFAULT
self.obj.get("DEFAULT_ENV_FOR_DYNACONF").lower(),
# default active env unless ENV_FOR_DYNACONF is changed
"development",
# backwards compatibility for global
"dynaconf",
# global that rules all
"global",
]
for env in envs:
env = env.lower() # lower for better comparison
data = {}
try:
data = source_data[env] or {}
except KeyError:
if env not in base_envs:
message = "%s_loader: %s env not defined in %s" % (
self.identifier,
env,
source_file,
)
if silent:
self.obj.logger.warning(message)
else:
raise KeyError(message)
continue
if env != self.obj.get("DEFAULT_ENV_FOR_DYNACONF").lower():
identifier = "{0}_{1}".format(self.identifier, env)
else:
identifier = self.identifier
# data 1st level keys should be transformed to upper case.
data = {upperfy(k): v for k, v in data.items()}
if key:
key = upperfy(key)
is_secret = "secret" in source_file
self.obj.logger.debug("{}_loader: {}[{}]{}".format(
self.identifier,
os.path.split(source_file)[-1],
env,
list(data.keys()) if is_secret else data,
))
# is there a `dynaconf_merge` inside an `[env]`?
file_merge = file_merge or data.pop("DYNACONF_MERGE", False)
if not key:
self.obj.update(
data,
loader_identifier=identifier,
is_secret=is_secret,
merge=file_merge,
)
elif key in data:
self.obj.set(
key,
data.get(key),
loader_identifier=identifier,
is_secret=is_secret,
merge=file_merge,
)
def get(self, item, default=None, *args, **kwargs):
value = super(DynaBox, self).get(item, default, *args, **kwargs)
if value is None or value == default:
n_item = item.lower() if item.isupper() else upperfy(item)
value = super(DynaBox, self).get(n_item, default, *args, **kwargs)
return value
def get(self, item, default=None, *args, **kwargs):
if item not in self: # toggle case
item = item.lower() if item.isupper() else upperfy(item)
return super(DynaBox, self).get(item, default, *args, **kwargs)
def __getitem__(self, item, *args, **kwargs):
try:
return super(DynaBox, self).__getitem__(item, *args, **kwargs)
except (AttributeError, KeyError):
n_item = item.lower() if item.isupper() else upperfy(item)
return super(DynaBox, self).__getitem__(n_item, *args, **kwargs)
