def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'))
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_write_to_bucket()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if '(NoSuchBucket)' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
conn.client.create_bucket(Bucket=conn.bucket)
print('Bucket created.')
else:
raise
def build_config(cls, new_config=None):
"""Build ConfigManager using environment and overrides."""
new_config = new_config or {}
config_manager = ConfigManager(
environments=[ConfigDictEnv(new_config),
ConfigOSEnv()])
return config_manager
def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'), no_verify=True)
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_configuration()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if 'HeadBucket operation: Not Found' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
conn._create_bucket()
print('Bucket created.')
else:
raise
def main(args, config=None):
if config is None:
config = ConfigManager(
environments=[
# Pull configuration from env file specified as JANSKY_ENV
ConfigEnvFileEnv([os.environ.get('JANSKY_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=('For configuration help, see '
'https://jansky.readthedocs.io/en/latest/configuration.html'))
app_config = AppConfig(config)
# Set a Sentry client if we're so configured
set_sentry_client(app_config('secret_sentry_dsn'), app_config('basedir'))
# Set up logging and sentry first, so we have something to log to. Then
# build and log everything else.
setup_logging(app_config)
# Log application configuration
log_config(logger, app_config)
# Set up Sentry exception logger if we're so configured
setup_sentry_logging()
# Set up metrics
setup_metrics(app_config('metrics_class'), config, logger)
# FIXME(willkg): run processor
print('Nothing to do, yet.')
def get_config():
return ConfigManager([
ConfigIniEnv([
os.environ.get('THREATRESPONSE_INI'), '~/.threatresponse.ini',
'/etc/threatresponse.ini'
]),
ConfigOSEnv()
])
def test_ConfigOSEnv():
os.environ["EVERETT_TEST_FOO"] = "bar"
os.environ["EVERETT_TEST_FOO"] = "bar"
cose = ConfigOSEnv()
assert cose.get("everett_test_foo") == "bar"
assert cose.get("EVERETT_test_foo") == "bar"
assert cose.get("foo", namespace=["everett", "test"]) == "bar"
def test_ConfigOSEnv():
os.environ['EVERETT_TEST_FOO'] = 'bar'
os.environ['EVERETT_TEST_FOO'] = 'bar'
cose = ConfigOSEnv()
assert cose.get('everett_test_foo') == 'bar'
assert cose.get('EVERETT_test_foo') == 'bar'
assert cose.get('foo', namespace=['everett', 'test']) == 'bar'
def get_config():
return ConfigManager([
ConfigIniEnv([
os.environ.get("CIS_CONFIG_INI"), "~/.mozilla-cis.ini",
"/etc/mozilla-cis.ini"
]),
ConfigOSEnv()
])
def build_config_manager():
config_manager = ConfigManager(
environments=[
# Pull configuration from environment variables
ConfigOSEnv()
],
doc="For configuration help, see https://ichnaea.readthedocs.io/",
)
return AppConfig(config_manager)
def __init__(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
# Set logging level
root_log = logging.getLogger('rastervision')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
help_doc = ('Check https://docs.rastervision.io/ for docs.')
self.config = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
[
# Allow overrides
ConfigDictEnv(config_overrides),
# Looks in OS environment first
ConfigOSEnv(),
# Look for an .env file
ConfigEnvFileEnv('.env'),
# Looks in INI files in order specified
ConfigIniEnv(config_file_locations),
],
# Make it easy for users to find your configuration docs
doc=help_doc)
def build_config_manager():
config = ConfigManager(
environments=[
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=('For configuration help, see '
'https://antenna.readthedocs.io/en/latest/configuration.html'))
return config
def get_config():
"""
Environment/yml config vars:
API_AUDIENCE
AUTHO_URL
"""
# load our config file (if any)
conf = yaml.load(open(os.environ.get("CONFIGFILE", "/dev/null")))
if conf is None:
conf = dict()
return ConfigManager([ConfigOSEnv(), ConfigDictEnv(conf)])
def getEverettManager(configfile):
return ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
[
# Looks in OS environment first
ConfigOSEnv(),
# Looks in INI files in order specified
ConfigIniEnv([
configfile,
]),
], )
def get_app(config=None):
"""Returns AntennaAPI instance"""
if config is None:
config = ConfigManager(
environments=[
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=(
'For configuration help, see '
'https://antenna.readthedocs.io/en/latest/configuration.html'
)
)
app_config = AppConfig(config)
# Set a Sentry client if we're so configured
set_sentry_client(app_config('secret_sentry_dsn'), app_config('basedir'))
# Set up logging and sentry first, so we have something to log to. Then
# build and log everything else.
setup_logging(app_config)
# Log application configuration
log_config(logger, app_config)
# Set up Sentry exception logger if we're so configured
setup_sentry_logging()
# Set up metrics
setup_metrics(app_config('metrics_class'), config, logger)
# Build the app, add resources, and log the rest of things
app = AntennaAPI(config)
app.add_route('breakpad', '/submit', BreakpadSubmitterResource(config))
app.add_route('version', '/__version__', VersionResource(config, basedir=app_config('basedir')))
app.add_route('heartbeat', '/__heartbeat__', HeartbeatResource(config, app))
app.add_route('lbheartbeat', '/__lbheartbeat__', LBHeartbeatResource(config))
app.add_route('broken', '/__broken__', BrokenResource(config))
log_config(logger, app)
# Wrap the app in some kind of unhandled exception notification mechanism
app = wsgi_capture_exceptions(app)
return app
def get_config(config_file=None):
"""Loads the configuration
Loads either the user supplied configuration, the configuration in the XDG
path, or the default config. Configuration may be given incompletely, so if
you only supply the color (for example), other configuration values are
taken from the defaults. The user can also supply a configuration as a
dictionary as an argument to this function, this takes first priority.
Parameters
----------
config_file : str or Path
Which config to load
Returns
-------
config : dict
The configuration to use
Example
-------
>>> config = get_config()
>>> debug = config.get("debug") # Evaluates to whatever debug is set in
# the first configuration found
"""
environments = [
# Look in OS process environment first
ConfigOSEnv(),
# Look in YAML files in order specified
ConfigYamlEnv(CONFIG_FILES),
]
if config_file:
environments.insert(0, config_file)
manager = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
environments=environments,
# Provide users a link to documentation for when they hit
# configuration errors
doc="Check https://example.com/configuration for docs.",
)
# Apply the configuration class to the configuration manager
# so that it handles option properties like defaults, parsers,
# documentation, and so on.
return manager.with_options(AppConfig())
def main():
config = ConfigManager(
environments=[ConfigOSEnv()],
doc='For configuration help, see https://example.com/configuration')
debug_mode = config(
'debug',
default='false',
parser=bool,
doc='True to put the app in debug mode. Don\'t use this in production!'
)
if debug_mode:
print('Debug mode is on!')
else:
print('Debug mode off.')
def main():
config = ConfigManager(
environments=[ConfigOSEnv()],
doc="For configuration help, see https://example.com/configuration",
)
debug_mode = config(
"debug",
default="false",
parser=bool,
doc="True to put the app in debug mode. Don't use this in production!",
)
if debug_mode:
print("Debug mode is on!")
else:
print("Debug mode off.")
def read_config(config_path: str):
"""
Read the configuration from the config ini file given.
:return:
"""
return ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
environments=[
# Look in OS process environment first
ConfigOSEnv(),
# Look in INI file given in parameter
ConfigIniEnv([config_path]),
],
# Provide users a link to documentation for when they hit
# configuration errors
doc='Check https://example.com/configuration for docs.')
def reset(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
root_log = logging.getLogger('rastervision2')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
self.config = ConfigManager(
[
ConfigDictEnv(config_overrides),
ConfigOSEnv(),
ConfigIniEnv(config_file_locations),
],
doc='Check https://docs.rastervision.io/ for docs.')
def test_dummy():
config = ConfigManager([
ConfigYamlEnv('my.yaml'),
ConfigOSEnv(),
ConfigDictEnv({'COMESFROMDICT': 'comes from dict'})
])
assert config('comesfromyaml') == 'comes from yaml'
assert config('THIS_COMES_FROM_YAML') == 'too'
assert config.with_namespace('this').with_namespace(
'comes').with_namespace('from')('yaml') == 'too'
assert config('USER') == 'aagibalov'
assert config('comesfromdict') == 'comes from dict'
try:
config('SomeSecret')
pytest.fail()
except everett.ConfigurationError:
pass
assert config('SomeSecret', default='SomeDefault') == 'SomeDefault'
def load_config(config_file):
# python-dotenv enables interpolation of values in config file
# from the environment or elsewhere in the config file using
# POSIX variable expansion
config_wrappers = []
config_wrappers.append(ConfigOSEnv())
local_conf = dotenv_values(find_dotenv())
config_wrappers.append(ConfigDictEnv(local_conf))
if config_file:
config_file_path = path.abspath(config_file)
project_conf = dotenv_values(dotenv_path=config_file_path)
config_wrappers.append(ConfigDictEnv(project_conf))
config_wrappers.append(ConfigDictEnv(settings.defaults()))
config = ConfigManager(config_wrappers)
return config
def __init__(self, config_map):
self.config_map = config_map
self.override = {} # type: Dict[str, Any]
self.backend_override = ConfigDictEnv({})
config_override = os.environ.get("COMET_INI")
if config_override is not None:
log_once_at_level(logging.WARNING,
"COMET_INI is deprecated; use COMET_CONFIG")
else:
config_override = os.environ.get("COMET_CONFIG")
self.manager = ConfigManager(
[ # User-defined overrides
ConfigOSEnv(),
ConfigEnvFileEnv(".env"),
ConfigIniEnv(config_override),
ConfigIniEnv("./.comet.config"),
ConfigIniEnv("~/.comet.config"),
# Comet-defined overrides
self.backend_override,
],
doc=
("See https://comet.ml/docs/python-sdk/getting-started/ for more "
+ "information on configuration."),
)
builds = len([
locale for locale, build in pd.firefox_primary_builds.items()
if version in build
])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data():
for key in FIREFOX_VERSION_KEYS:
count_builds(key)
# setup and run
config = ConfigManager([ConfigOSEnv(), ConfigEnvFileEnv(path('.env'))])
settings.configure(
DEBUG=False,
PROD_DETAILS_DIR=path('product-details'),
PROD_DETAILS_URL=config(
'PROD_DETAILS_URL',
default='https://product-details.mozilla.org/1.0/'),
PROD_DETAILS_STORAGE='product_details.storage.PDFileStorage',
INSTALLED_APPS=['product_details'],
# disable cache
CACHES={
'default': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
}
},
)
def get_program_config():
sources = [ConfigOSEnv()]
if "RS_CONFIG_FILE" in os.environ:
sources.append(ConfigYamlEnv(os.environ["RS_CONFIG_FILE"]))
config = ConfigManager(sources)
return MainConfig(config.with_namespace("rs"))
from pathlib import Path
from everett.manager import ConfigManager, ConfigEnvFileEnv, ConfigOSEnv, ListOf
import django_cache_url
ROOT_PATH = Path(__file__).resolve().parents[1]
def path(*paths):
return str(ROOT_PATH.joinpath(*paths))
config = ConfigManager([
ConfigEnvFileEnv(path('.env')),
ConfigOSEnv(),
])
# Build paths inside the project like this: path(...)
BASE_DIR = str(ROOT_PATH)
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default='false', parser=bool)
ALLOWED_HOSTS = config('ALLOWED_HOSTS',
parser=ListOf(str),
def set_everett_config(self,
profile: str = None,
rv_home: str = None,
config_overrides: Dict[str, str] = None):
"""Set Everett config.
This sets up any other configuration using the Everett library.
See https://everett.readthedocs.io/
It roughly mimics the behavior of how the AWS CLI is configured, if that
is a helpful analogy. Configuration can be specified through configuration
files, environment variables, and the config_overrides argument in increasing
order of precedence.
Configuration files are in the following format:
```
[namespace_1]
key_11=val_11
...
key_1n=val_1n
...
[namespace_m]
key_m1=val_m1
...
key_mn=val_mn
```
Each namespace can be used for the configuration of a different plugin.
Each configuration file is a "profile" with the name of the file being the name
of the profile. This supports switching between different configuration sets.
The corresponding environment variable setting for namespace_i and key_ij is
`namespace_i_key_ij=val_ij`.
Args:
profile: name of the RV configuration profile to use. If not set, defaults
to value of RV_PROFILE env var, or DEFAULT_PROFILE.
rv_home: a local dir with RV configuration files. If not set, attempts to
use ~/.rastervision.
config_overrides: any configuration to override. Each key is of form
namespace_i_key_ij with corresponding value val_ij.
"""
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
self.profile = profile
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
if self.profile == 'local':
config_ini_env = LocalEnv()
else:
config_file_locations = self._discover_config_file_locations(
self.profile)
config_ini_env = ConfigIniEnv(config_file_locations)
self.config = ConfigManager(
[
ConfigDictEnv(config_overrides),
ConfigOSEnv(),
config_ini_env,
],
doc=(
'Check https://docs.rastervision.io/ for docs. '
'Switch to the version being run and search for Raster Vision '
'Configuration.'))
def get_config():
return ConfigManager([ConfigOSEnv()])
def config():
cfg = ConfigManager([ConfigOSEnv()])
return cfg
def get_config():
config = ConfigManager(environments=[ConfigOSEnv()])
return config.with_options(BotConfig())
