def test_ConfigDictEnv():
cde = ConfigDictEnv({
'FOO': 'bar',
'A_FOO': 'a_bar',
'A_B_FOO': 'a_b_bar',
})
assert cde.get('foo') == 'bar'
assert cde.get('foo', namespace=['a']) == 'a_bar'
assert cde.get('foo', namespace=['a', 'b']) == 'a_b_bar'
def test_with_options():
"""Verify .with_options() restricts configuration"""
config = ConfigManager([
ConfigDictEnv({
"FOO_BAR": "a",
"FOO_BAZ": "b",
"BAR": "c",
"BAZ": "d"
})
])
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("baz",
default="",
doc="some help here",
parser=str)
def __init__(self, config):
self.config = config.with_options(self)
# Create the component with regular config
comp = SomeComponent(config)
assert comp.config("baz") == "d"
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp.config("bar")
# Create the component with config in the "foo" namespace
comp2 = SomeComponent(config.with_namespace("foo"))
assert comp2.config("baz") == "b"
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp2.config("bar")
def test_with_options():
"""Verify .with_options() restricts configuration"""
config = ConfigManager([
ConfigDictEnv({
'FOO_BAR': 'a',
'FOO_BAZ': 'b',
'BAR': 'c',
'BAZ': 'd',
})
])
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('baz',
default='',
doc='some help here',
parser=str)
def __init__(self, config):
self.config = config.with_options(self)
# Create the component with regular config
comp = SomeComponent(config)
assert comp.config('baz') == 'd'
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp.config('bar')
# Create the component with config in the "foo" namespace
comp2 = SomeComponent(config.with_namespace('foo'))
assert comp2.config('baz') == 'b'
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp2.config('bar')
def build_config(cls, new_config=None):
"""Build ConfigManager using environment and overrides."""
new_config = new_config or {}
config_manager = ConfigManager(
environments=[ConfigDictEnv(new_config),
ConfigOSEnv()])
return config_manager
def test_config_manager_doc():
config = ConfigManager([
ConfigDictEnv({'foo': 'bar'}),
],
doc='See http://example.com/configuration')
# Test ConfigManager doc shows up
if six.PY3:
with pytest.raises(ConfigurationError) as exc_info:
config('foo', parser=int)
else:
with pytest.raises(ValueError) as exc_info:
config('foo', parser=int)
assert (str(exc_info.value) ==
'ValueError: invalid literal for int() with base 10: \'bar\'\n'
'namespace=None key=foo requires a value parseable by int\n'
'See http://example.com/configuration')
# Test config doc and ConfigManager doc show up
if six.PY3:
with pytest.raises(ConfigurationError) as exc_info:
config('foo', parser=int, doc='Port to listen on.')
else:
with pytest.raises(ValueError) as exc_info:
config('foo', parser=int, doc='Port to listen on.')
assert (str(exc_info.value) ==
'ValueError: invalid literal for int() with base 10: \'bar\'\n'
'namespace=None key=foo requires a value parseable by int\n'
'Port to listen on.\n'
'See http://example.com/configuration')
def test_ConfigDictEnv():
cde = ConfigDictEnv({
"FOO": "bar",
"A_FOO": "a_bar",
"A_B_FOO": "a_b_bar",
"lower_foo": "bar"
})
assert cde.get("foo") == "bar"
assert cde.get("foo", namespace=["a"]) == "a_bar"
assert cde.get("foo", namespace=["a", "b"]) == "a_b_bar"
assert cde.get("FOO", namespace=["a"]) == "a_bar"
assert cde.get("foo", namespace=["A"]) == "a_bar"
assert cde.get("FOO", namespace=["A"]) == "a_bar"
cde = ConfigDictEnv({"foo": "bar"})
assert cde.get("foo") == "bar"
assert cde.get("FOO") == "bar"
def __init__(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
# Set logging level
root_log = logging.getLogger('rastervision')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
help_doc = ('Check https://docs.rastervision.io/ for docs.')
self.config = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
[
# Allow overrides
ConfigDictEnv(config_overrides),
# Looks in OS environment first
ConfigOSEnv(),
# Look for an .env file
ConfigEnvFileEnv('.env'),
# Looks in INI files in order specified
ConfigIniEnv(config_file_locations),
],
# Make it easy for users to find your configuration docs
doc=help_doc)
def test_ConfigDictEnv():
cde = ConfigDictEnv({
'FOO': 'bar',
'A_FOO': 'a_bar',
'A_B_FOO': 'a_b_bar',
'lower_foo': 'bar'
})
assert cde.get('foo') == 'bar'
assert cde.get('foo', namespace=['a']) == 'a_bar'
assert cde.get('foo', namespace=['a', 'b']) == 'a_b_bar'
assert cde.get('FOO', namespace=['a']) == 'a_bar'
assert cde.get('foo', namespace=['A']) == 'a_bar'
assert cde.get('FOO', namespace=['A']) == 'a_bar'
cde = ConfigDictEnv({
'foo': 'bar',
})
assert cde.get('foo') == 'bar'
assert cde.get('FOO') == 'bar'
def get_config():
"""
Environment/yml config vars:
API_AUDIENCE
AUTHO_URL
"""
# load our config file (if any)
conf = yaml.load(open(os.environ.get("CONFIGFILE", "/dev/null")))
if conf is None:
conf = dict()
return ConfigManager([ConfigOSEnv(), ConfigDictEnv(conf)])
def load_config(config_file):
# python-dotenv enables interpolation of values in config file
# from the environment or elsewhere in the config file using
# POSIX variable expansion
config_wrappers = []
config_wrappers.append(ConfigOSEnv())
local_conf = dotenv_values(find_dotenv())
config_wrappers.append(ConfigDictEnv(local_conf))
if config_file:
config_file_path = path.abspath(config_file)
project_conf = dotenv_values(dotenv_path=config_file_path)
config_wrappers.append(ConfigDictEnv(project_conf))
config_wrappers.append(ConfigDictEnv(settings.defaults()))
config = ConfigManager(config_wrappers)
return config
def test_parser_comes_from_options():
"""Verify the parser is picked up from options"""
config = ConfigManager([ConfigDictEnv({'FOO': '1'})])
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('foo', parser=int)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config)
assert comp.config('foo') == 1
def config_from_dict(d):
def flatten_dict(d, prefix=""):
newd = {}
for key, val in d.items():
if isinstance(val, dict):
flatval = flatten_dict(val, f"{prefix}{key}_")
newd.update(flatval)
else:
newd[f"{prefix}{key}"] = val
return newd
flatd = flatten_dict(d)
return ConfigManager([ConfigDictEnv(flatd)])
def reset(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
root_log = logging.getLogger('rastervision2')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
self.config = ConfigManager(
[
ConfigDictEnv(config_overrides),
ConfigOSEnv(),
ConfigIniEnv(config_file_locations),
],
doc='Check https://docs.rastervision.io/ for docs.')
def test_dummy():
config = ConfigManager([
ConfigYamlEnv('my.yaml'),
ConfigOSEnv(),
ConfigDictEnv({'COMESFROMDICT': 'comes from dict'})
])
assert config('comesfromyaml') == 'comes from yaml'
assert config('THIS_COMES_FROM_YAML') == 'too'
assert config.with_namespace('this').with_namespace(
'comes').with_namespace('from')('yaml') == 'too'
assert config('USER') == 'aagibalov'
assert config('comesfromdict') == 'comes from dict'
try:
config('SomeSecret')
pytest.fail()
except everett.ConfigurationError:
pass
assert config('SomeSecret', default='SomeDefault') == 'SomeDefault'
def test_with_namespace():
config = ConfigManager(
[ConfigDictEnv({
'FOO_BAR': 'foobaz',
'BAR': 'baz',
'BAT': 'bat',
})])
# Verify the values first
assert config('bar', namespace=['foo']) == 'foobaz'
assert config('bar') == 'baz'
assert config('bat') == 'bat'
# Create the namespaced config
config_with_namespace = config.with_namespace('foo')
assert config_with_namespace('bar') == 'foobaz'
# Verify 'bat' is not available because it's not in the namespace
with pytest.raises(ConfigurationError):
config_with_namespace('bat')
def test_with_namespace():
config = ConfigManager(
[ConfigDictEnv({
"FOO_BAR": "foobaz",
"BAR": "baz",
"BAT": "bat"
})])
# Verify the values first
assert config("bar", namespace=["foo"]) == "foobaz"
assert config("bar") == "baz"
assert config("bat") == "bat"
# Create the namespaced config
config_with_namespace = config.with_namespace("foo")
assert config_with_namespace("bar") == "foobaz"
# Verify 'bat' is not available because it's not in the namespace
with pytest.raises(ConfigurationError):
config_with_namespace("bat")
def test_config_manager_doc():
config = ConfigManager([ConfigDictEnv({"foo": "bar"})],
doc="See http://example.com/configuration")
# Test ConfigManager doc shows up
with pytest.raises(ConfigurationError) as exc_info:
config("foo", parser=int)
assert (str(exc_info.value) ==
"ValueError: invalid literal for int() with base 10: 'bar'\n"
"namespace=None key=foo requires a value parseable by int\n"
"See http://example.com/configuration")
# Test config doc and ConfigManager doc show up
with pytest.raises(ConfigurationError) as exc_info:
config("foo", parser=int, doc="Port to listen on.")
assert (str(exc_info.value) ==
"ValueError: invalid literal for int() with base 10: 'bar'\n"
"namespace=None key=foo requires a value parseable by int\n"
"Port to listen on.\n"
"See http://example.com/configuration")
def __init__(self, config_map):
self.config_map = config_map
self.override = {} # type: Dict[str, Any]
self.backend_override = ConfigDictEnv({})
config_override = os.environ.get("COMET_INI")
if config_override is not None:
log_once_at_level(logging.WARNING,
"COMET_INI is deprecated; use COMET_CONFIG")
else:
config_override = os.environ.get("COMET_CONFIG")
self.manager = ConfigManager(
[ # User-defined overrides
ConfigOSEnv(),
ConfigEnvFileEnv(".env"),
ConfigIniEnv(config_override),
ConfigIniEnv("./.comet.config"),
ConfigIniEnv("~/.comet.config"),
# Comet-defined overrides
self.backend_override,
],
doc=
("See https://comet.ml/docs/python-sdk/getting-started/ for more "
+ "information on configuration."),
)
def set_everett_config(self,
profile: str = None,
rv_home: str = None,
config_overrides: Dict[str, str] = None):
"""Set Everett config.
This sets up any other configuration using the Everett library.
See https://everett.readthedocs.io/
It roughly mimics the behavior of how the AWS CLI is configured, if that
is a helpful analogy. Configuration can be specified through configuration
files, environment variables, and the config_overrides argument in increasing
order of precedence.
Configuration files are in the following format:
```
[namespace_1]
key_11=val_11
...
key_1n=val_1n
...
[namespace_m]
key_m1=val_m1
...
key_mn=val_mn
```
Each namespace can be used for the configuration of a different plugin.
Each configuration file is a "profile" with the name of the file being the name
of the profile. This supports switching between different configuration sets.
The corresponding environment variable setting for namespace_i and key_ij is
`namespace_i_key_ij=val_ij`.
Args:
profile: name of the RV configuration profile to use. If not set, defaults
to value of RV_PROFILE env var, or DEFAULT_PROFILE.
rv_home: a local dir with RV configuration files. If not set, attempts to
use ~/.rastervision.
config_overrides: any configuration to override. Each key is of form
namespace_i_key_ij with corresponding value val_ij.
"""
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
self.profile = profile
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
if self.profile == 'local':
config_ini_env = LocalEnv()
else:
config_file_locations = self._discover_config_file_locations(
self.profile)
config_ini_env = ConfigIniEnv(config_file_locations)
self.config = ConfigManager(
[
ConfigDictEnv(config_overrides),
ConfigOSEnv(),
config_ini_env,
],
doc=(
'Check https://docs.rastervision.io/ for docs. '
'Switch to the version being run and search for Raster Vision '
'Configuration.'))
import os
from everett.manager import (ConfigDictEnv, ConfigIniEnv, ConfigManager,
ConfigOSEnv)
config = ConfigManager([
# Pull from the OS environment first
ConfigOSEnv(),
# Fall back to the file specified by the FOO_INI OS environment
# variable if such file exists
ConfigIniEnv(os.environ.get('FOO_INI')),
# Fall back to this dict of defaults
ConfigDictEnv({'FOO_VAR': 'bar'})
])
assert config('foo_var') == 'bar'
import os
import raven
from everett.manager import (
ConfigDictEnv,
ConfigEnvFileEnv,
ConfigIniEnv,
ConfigManager,
ConfigOSEnv,
ListOf,
)
config = ConfigManager([
ConfigOSEnv(),
ConfigIniEnv(['config.ini', '/etc/signoxe/server.ini']),
ConfigDictEnv({
'SECRET_KEY': 'vo7v#s9o7$t%x=fpbt7j5#%=-bl^y6e4&n2hpklg&rzx%z2mp$',
'DEBUG': 'false',
})
])
# Core config
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ADMINS = (('Kshitij Sobti', '*****@*****.**'), )
DEBUG = config('DEBUG', parser=bool)
SECRET_KEY = config('SECRET_KEY')
ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='', parser=ListOf(str))
import os
from everett.ext.inifile import ConfigIniEnv
from everett.manager import ConfigDictEnv, ConfigManager, ConfigOSEnv
config = ConfigManager([
# Pull from the OS environment first
ConfigOSEnv(),
# Fall back to the file specified by the FOO_INI OS environment
# variable if such file exists
ConfigIniEnv(os.environ.get("FOO_INI")),
# Fall back to this dict of defaults
ConfigDictEnv({"FOO_VAR": "bar"}),
])
assert config("foo_var") == "bar"