def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'), no_verify=True)
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_configuration()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if 'HeadBucket operation: Not Found' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
conn._create_bucket()
print('Bucket created.')
else:
raise
def main(args, config=None):
if config is None:
config = ConfigManager(
environments=[
# Pull configuration from env file specified as JANSKY_ENV
ConfigEnvFileEnv([os.environ.get('JANSKY_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=('For configuration help, see '
'https://jansky.readthedocs.io/en/latest/configuration.html'))
app_config = AppConfig(config)
# Set a Sentry client if we're so configured
set_sentry_client(app_config('secret_sentry_dsn'), app_config('basedir'))
# Set up logging and sentry first, so we have something to log to. Then
# build and log everything else.
setup_logging(app_config)
# Log application configuration
log_config(logger, app_config)
# Set up Sentry exception logger if we're so configured
setup_sentry_logging()
# Set up metrics
setup_metrics(app_config('metrics_class'), config, logger)
# FIXME(willkg): run processor
print('Nothing to do, yet.')
def __init__(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
# Set logging level
root_log = logging.getLogger('rastervision')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
help_doc = ('Check https://docs.rastervision.io/ for docs.')
self.config = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
[
# Allow overrides
ConfigDictEnv(config_overrides),
# Looks in OS environment first
ConfigOSEnv(),
# Look for an .env file
ConfigEnvFileEnv('.env'),
# Looks in INI files in order specified
ConfigIniEnv(config_file_locations),
],
# Make it easy for users to find your configuration docs
doc=help_doc)
def get_app(config=None):
"""Returns AntennaAPI instance"""
if config is None:
config = ConfigManager(
environments=[
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=(
'For configuration help, see '
'https://antenna.readthedocs.io/en/latest/configuration.html'
)
)
app_config = AppConfig(config)
# Set a Sentry client if we're so configured
set_sentry_client(app_config('secret_sentry_dsn'), app_config('basedir'))
# Set up logging and sentry first, so we have something to log to. Then
# build and log everything else.
setup_logging(app_config)
# Log application configuration
log_config(logger, app_config)
# Set up Sentry exception logger if we're so configured
setup_sentry_logging()
# Set up metrics
setup_metrics(app_config('metrics_class'), config, logger)
# Build the app, add resources, and log the rest of things
app = AntennaAPI(config)
app.add_route('breakpad', '/submit', BreakpadSubmitterResource(config))
app.add_route('version', '/__version__', VersionResource(config, basedir=app_config('basedir')))
app.add_route('heartbeat', '/__heartbeat__', HeartbeatResource(config, app))
app.add_route('lbheartbeat', '/__lbheartbeat__', LBHeartbeatResource(config))
app.add_route('broken', '/__broken__', BrokenResource(config))
log_config(logger, app)
# Wrap the app in some kind of unhandled exception notification mechanism
app = wsgi_capture_exceptions(app)
return app
def test_ConfigEnvFileEnv(datadir):
env_filename = os.path.join(datadir, '.env')
cefe = ConfigEnvFileEnv(['/does/not/exist/.env', env_filename])
assert cefe.get('not_a', namespace='youre') == 'golfer'
assert cefe.get('loglevel') == 'walter'
assert cefe.get('LOGLEVEL') == 'walter'
assert cefe.get('missing') is NO_VALUE
assert cefe.data == {
'LOGLEVEL': 'walter',
'DEBUG': 'True',
'YOURE_NOT_A': 'golfer',
'DATABASE_URL': 'sqlite:///kahlua.db',
}
cefe = ConfigEnvFileEnv(env_filename)
assert cefe.get('not_a', namespace='youre') == 'golfer'
cefe = ConfigEnvFileEnv('/does/not/exist/.env')
assert cefe.get('loglevel') is NO_VALUE
def __init__(self, config_map):
self.config_map = config_map
self.override = {} # type: Dict[str, Any]
self.backend_override = ConfigDictEnv({})
config_override = os.environ.get("COMET_INI")
if config_override is not None:
log_once_at_level(logging.WARNING,
"COMET_INI is deprecated; use COMET_CONFIG")
else:
config_override = os.environ.get("COMET_CONFIG")
self.manager = ConfigManager(
[ # User-defined overrides
ConfigOSEnv(),
ConfigEnvFileEnv(".env"),
ConfigIniEnv(config_override),
ConfigIniEnv("./.comet.config"),
ConfigIniEnv("~/.comet.config"),
# Comet-defined overrides
self.backend_override,
],
doc=
("See https://comet.ml/docs/python-sdk/getting-started/ for more "
+ "information on configuration."),
)
builds = len([
locale for locale, build in pd.firefox_primary_builds.items()
if version in build
])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data():
for key in FIREFOX_VERSION_KEYS:
count_builds(key)
# setup and run
config = ConfigManager([ConfigOSEnv(), ConfigEnvFileEnv(path('.env'))])
settings.configure(
DEBUG=False,
PROD_DETAILS_DIR=path('product-details'),
PROD_DETAILS_URL=config(
'PROD_DETAILS_URL',
default='https://product-details.mozilla.org/1.0/'),
PROD_DETAILS_STORAGE='product_details.storage.PDFileStorage',
INSTALLED_APPS=['product_details'],
# disable cache
CACHES={
'default': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
}
},
)
from pathlib import Path
from everett.manager import ConfigManager, ConfigEnvFileEnv, ConfigOSEnv, ListOf
import django_cache_url
ROOT_PATH = Path(__file__).resolve().parents[1]
def path(*paths):
return str(ROOT_PATH.joinpath(*paths))
config = ConfigManager([
ConfigEnvFileEnv(path('.env')),
ConfigOSEnv(),
])
# Build paths inside the project like this: path(...)
BASE_DIR = str(ROOT_PATH)
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default='false', parser=bool)
ALLOWED_HOSTS = config('ALLOWED_HOSTS',
parser=ListOf(str),
def test_ConfigEnvFileEnv(datadir):
env_filename = os.path.join(datadir, '.env')
cefe = ConfigEnvFileEnv(['/does/not/exist/.env', env_filename])
assert cefe.get('not_a', namespace='youre') == 'golfer'
assert cefe.get('loglevel') == 'walter'
assert cefe.get('LOGLEVEL') == 'walter'
assert cefe.get('missing') is NO_VALUE
assert cefe.data == {
'LOGLEVEL': 'walter',
'DEBUG': 'True',
'YOURE_NOT_A': 'golfer',
'DATABASE_URL': 'sqlite:///kahlua.db',
}
cefe = ConfigEnvFileEnv(env_filename)
assert cefe.get('not_a', namespace='youre') == 'golfer'
cefe = ConfigEnvFileEnv('/does/not/exist/.env')
assert cefe.get('loglevel') is NO_VALUE
def test_ConfigEnvFileEnv(datadir):
env_filename = os.path.join(datadir, ".env")
cefe = ConfigEnvFileEnv(["/does/not/exist/.env", env_filename])
assert cefe.get("not_a", namespace="youre") == "golfer"
assert cefe.get("loglevel") == "walter"
assert cefe.get("LOGLEVEL") == "walter"
assert cefe.get("missing") is NO_VALUE
assert cefe.data == {
"LOGLEVEL": "walter",
"DEBUG": "True",
"YOURE_NOT_A": "golfer",
"DATABASE_URL": "sqlite:///kahlua.db",
}
cefe = ConfigEnvFileEnv(env_filename)
assert cefe.get("not_a", namespace="youre") == "golfer"
cefe = ConfigEnvFileEnv("/does/not/exist/.env")
assert cefe.get("loglevel") is NO_VALUE
def run_program(app, args, parser=None):
if parser is None:
parser = argparse.ArgumentParser(prog=app.program_name)
parser.add_argument(
'--config',
help=('Config file in ENV format. Setting options on the command line '
'will override values in the config file.'),
)
options = app.get_required_config()
for opt in options:
# We don't enforce required here--we do that in a later pass so we can
# take configuration files into account.
kwargs = {
'help': opt.doc,
'type': handle_no_value(opt.parser),
'action': 'store',
}
if opt.default is not NO_VALUE:
kwargs['help'] += ' Default is %s.' % opt.default
parser.add_argument('--%s' % opt.key.lower(), **kwargs)
parser.set_defaults(handler=app)
# Parse the args--this will exit if there's a --help
vals, extra = parser.parse_known_args(args)
config = ConfigManager([
ConfigObjEnv(vals),
# FIXME(willkg): It'd be better if this was an INI file.
ConfigEnvFileEnv(vals.config),
])
# Now go through and make sure all the required options were supplied.
missing_opts = []
parse_errors = []
comp_config = config.with_options(app)
for opt in options:
try:
comp_config(opt.key.lower())
except ConfigurationMissingError as cme:
missing_opts.append(opt.key.lower())
except Exception as exc:
parse_errors.append((opt.key.lower(), str(exc)))
if missing_opts:
parser.print_usage(sys.stderr)
print_error('the following are required: %s\n' %
', '.join([opt for opt in missing_opts]))
return 2
if parse_errors:
parser.print_usage(sys.stderr)
print_error('the following have value errors:\n')
for opt, msg in parse_errors:
print_error('%s:\n%s\n' % (opt, indent(msg)))
return 2
return vals.handler(config).invoke()
from everett.manager import (
ConfigEnvFileEnv,
ConfigManager,
ConfigOSEnv,
)
config = ConfigManager([
# first check for environment variables
ConfigOSEnv(),
# then look in the .env file
ConfigEnvFileEnv('.env'),
])
self.timeout = 300
self.last_update = 0
def get_cache(self):
if time() > self.last_update + self.timeout:
return None
return self._data
def set_cache(self, data):
self._data = data
self.last_update = time()
@property
def cfg(self):
# this is the method called by the get method
# of the superclass
configs = self.get_cache()
if not configs:
configs = get_config_dict()
self.set_cache(configs)
return configs
config = ConfigManager([
ConfigOSEnv(),
ConfigEnvFileEnv(".env"),
ConfigDBEnv(),
])
