def test_with_options():
"""Verify .with_options() restricts configuration"""
config = ConfigManager([
ConfigDictEnv({
"FOO_BAR": "a",
"FOO_BAZ": "b",
"BAR": "c",
"BAZ": "d"
})
])
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("baz",
default="",
doc="some help here",
parser=str)
def __init__(self, config):
self.config = config.with_options(self)
# Create the component with regular config
comp = SomeComponent(config)
assert comp.config("baz") == "d"
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp.config("bar")
# Create the component with config in the "foo" namespace
comp2 = SomeComponent(config.with_namespace("foo"))
assert comp2.config("baz") == "b"
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp2.config("bar")
def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'), no_verify=True)
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_configuration()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if 'HeadBucket operation: Not Found' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
# Create the bucket.
conn._create_bucket()
print('Bucket created.')
else:
raise
def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'))
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_write_to_bucket()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if '(NoSuchBucket)' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
conn.client.create_bucket(Bucket=conn.bucket)
print('Bucket created.')
else:
raise
def test_nested_options():
"""Verify nested BoundOptions works."""
config = ConfigManager.from_dict({})
class Foo(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option(
'option1',
default='opt1default',
parser=str
)
class Bar(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option(
'option2',
default='opt2default',
parser=str
)
config = ConfigManager.basic_config()
config = config.with_options(Foo)
config = config.with_options(Bar)
assert config('option2') == 'opt2default'
with pytest.raises(ConfigurationError):
config('option1')
def test_with_options():
"""Verify .with_options() restricts configuration"""
config = ConfigManager([
ConfigDictEnv({
'FOO_BAR': 'a',
'FOO_BAZ': 'b',
'BAR': 'c',
'BAZ': 'd',
})
])
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('baz',
default='',
doc='some help here',
parser=str)
def __init__(self, config):
self.config = config.with_options(self)
# Create the component with regular config
comp = SomeComponent(config)
assert comp.config('baz') == 'd'
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp.config('bar')
# Create the component with config in the "foo" namespace
comp2 = SomeComponent(config.with_namespace('foo'))
assert comp2.config('baz') == 'b'
with pytest.raises(ConfigurationError):
# This is not a valid option for this component
comp2.config('bar')
def main(args):
# Build configuration object just like we do in Antenna.
config = ConfigManager([
# Pull configuration from env file specified as ANTENNA_ENV
ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
])
# We create it in the crashstorage namespace because that's how Antenna
# uses it. This makes it easier to use existing configuration.
conn = S3Connection(config.with_namespace('crashstorage'), no_verify=True)
# First, check to see if the bucket is already created.
try:
print('Checking to see if bucket "%s" exists...' % conn.bucket)
conn.verify_configuration()
print('Bucket exists.')
except ClientError as exc:
print(str(exc))
if 'HeadBucket operation: Not Found' in str(exc):
print('Bucket not found. Creating %s ...' % conn.bucket)
conn._create_bucket()
print('Bucket created.')
else:
raise
def test_config_from_dict():
config = ConfigManager.from_dict({})
assert config("FOO", raise_error=False) is NO_VALUE
config = ConfigManager.from_dict({"FOO": "bar"})
assert config("FOO", raise_error=False) == "bar"
def test_config_from_dict():
config = ConfigManager.from_dict({})
assert config('FOO', raise_error=False) is NO_VALUE
config = ConfigManager.from_dict({'FOO': 'bar'})
assert config('FOO', raise_error=False) == 'bar'
def __init__(self,
profile=None,
rv_home=None,
config_overrides=None,
tmp_dir=None,
verbosity=Verbosity.NORMAL):
self.verbosity = verbosity
# Set logging level
root_log = logging.getLogger('rastervision')
if self.verbosity >= Verbosity.VERBOSE:
root_log.setLevel(logging.DEBUG)
elif self.verbosity >= Verbosity.NORMAL:
root_log.setLevel(logging.INFO)
else:
root_log.setLevel(logging.WARN)
if tmp_dir is not None:
self.set_tmp_dir(tmp_dir)
if profile is None:
if os.environ.get('RV_PROFILE'):
profile = os.environ.get('RV_PROFILE')
else:
profile = RVConfig.DEFAULT_PROFILE
if config_overrides is None:
config_overrides = {}
if rv_home is None:
home = os.path.expanduser('~')
rv_home = os.path.join(home, '.rastervision')
self.rv_home = rv_home
config_file_locations = self._discover_config_file_locations(profile)
help_doc = ('Check https://docs.rastervision.io/ for docs.')
self.config = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
[
# Allow overrides
ConfigDictEnv(config_overrides),
# Looks in OS environment first
ConfigOSEnv(),
# Look for an .env file
ConfigEnvFileEnv('.env'),
# Looks in INI files in order specified
ConfigIniEnv(config_file_locations),
],
# Make it easy for users to find your configuration docs
doc=help_doc)
def pytest_runtest_setup():
# Make sure we set up logging and metrics to sane default values.
setup_logging(ConfigManager.from_dict({
'HOST_ID': '',
'LOGGING_LEVEL': 'DEBUG',
}))
setup_metrics(metrics.LoggingMetrics, ConfigManager.from_dict({}))
# Wipe any registered heartbeat functions
reset_hb_funs()
def test_config_from_dict():
config = ConfigManager.from_dict({})
assert config('FOO', raise_error=False) is NO_VALUE
config = ConfigManager.from_dict({
'FOO': 'bar'
})
assert config('FOO', raise_error=False) == 'bar'
def test_with_argparse(self):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', help='to debug or not to debug')
parsed_vals = parser.parse_known_args([])[0]
config = ConfigManager([ConfigObjEnv(parsed_vals)])
assert config('debug', parser=bool, raise_error=False) is NO_VALUE
parsed_vals = parser.parse_known_args(['--debug=y'])[0]
config = ConfigManager([ConfigObjEnv(parsed_vals)])
assert config('debug', parser=bool) is True
def get_config(config_file=None):
"""Loads the configuration
Loads either the user supplied configuration, the configuration in the XDG
path, or the default config. Configuration may be given incompletely, so if
you only supply the color (for example), other configuration values are
taken from the defaults. The user can also supply a configuration as a
dictionary as an argument to this function, this takes first priority.
Parameters
----------
config_file : str or Path
Which config to load
Returns
-------
config : dict
The configuration to use
Example
-------
>>> config = get_config()
>>> debug = config.get("debug") # Evaluates to whatever debug is set in
# the first configuration found
"""
environments = [
# Look in OS process environment first
ConfigOSEnv(),
# Look in YAML files in order specified
ConfigYamlEnv(CONFIG_FILES),
]
if config_file:
environments.insert(0, config_file)
manager = ConfigManager(
# Specify one or more configuration environments in
# the order they should be checked
environments=environments,
# Provide users a link to documentation for when they hit
# configuration errors
doc="Check https://example.com/configuration for docs.",
)
# Apply the configuration class to the configuration manager
# so that it handles option properties like defaults, parsers,
# documentation, and so on.
return manager.with_options(AppConfig())
def test_is_fennec(self):
raw_crash = {
'ProductName': 'Fennec'
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_fennec', 100)
def test_is_firefox(self, randommock):
with randommock(0.09):
raw_crash = {
'ProductName': 'Firefox',
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_firefox_desktop', 10)
with randommock(0.9):
raw_crash = {
'ProductName': 'Firefox',
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (DEFER, 'is_firefox_desktop', 10)
def test_gauge(self):
metrics.metrics_configure(metrics.DogStatsdMetrics, ConfigManager.from_dict({}))
mymetrics = metrics.get_metrics('foobar')
with patch.object(metrics._metrics_impl.client, 'gauge') as mock_gauge:
mymetrics.gauge('key1', 5)
mock_gauge.assert_called_with(metric='foobar.key1', value=5)
def test_timing(self):
metrics.metrics_configure(metrics.DogStatsdMetrics, ConfigManager.from_dict({}))
mymetrics = metrics.get_metrics('foobar')
with patch.object(metrics._metrics_impl.client, 'timing') as mock_timing:
mymetrics.timing('key1', 1000)
mock_timing.assert_called_with(metric='foobar.key1', value=1000)
def test_incr(self):
metrics.metrics_configure(metrics.DogStatsdMetrics, ConfigManager.from_dict({}))
mymetrics = metrics.get_metrics('foobar')
with patch.object(metrics._metrics_impl.client, 'increment') as mock_incr:
mymetrics.incr('key1')
mock_incr.assert_called_with(metric='foobar.key1', value=1)
def test_config_manager_doc():
config = ConfigManager([
ConfigDictEnv({'foo': 'bar'}),
],
doc='See http://example.com/configuration')
# Test ConfigManager doc shows up
if six.PY3:
with pytest.raises(ConfigurationError) as exc_info:
config('foo', parser=int)
else:
with pytest.raises(ValueError) as exc_info:
config('foo', parser=int)
assert (str(exc_info.value) ==
'ValueError: invalid literal for int() with base 10: \'bar\'\n'
'namespace=None key=foo requires a value parseable by int\n'
'See http://example.com/configuration')
# Test config doc and ConfigManager doc show up
if six.PY3:
with pytest.raises(ConfigurationError) as exc_info:
config('foo', parser=int, doc='Port to listen on.')
else:
with pytest.raises(ValueError) as exc_info:
config('foo', parser=int, doc='Port to listen on.')
assert (str(exc_info.value) ==
'ValueError: invalid literal for int() with base 10: \'bar\'\n'
'namespace=None key=foo requires a value parseable by int\n'
'Port to listen on.\n'
'See http://example.com/configuration')
def test_parse_bool_with_config():
config = ConfigManager.from_dict({'foo': 'bar'})
# Test key is there, but value is bad
if six.PY3:
with pytest.raises(InvalidValueError) as excinfo:
config('foo', parser=bool)
else:
with pytest.raises(ValueError) as excinfo:
config('foo', parser=bool)
assert (
str(excinfo.value) == 'ValueError: "bar" is not a valid bool value\n'
'namespace=None key=foo requires a value parseable by everett.manager.parse_bool'
)
# Test key is not there and default is bad
if six.PY3:
with pytest.raises(InvalidValueError) as excinfo:
config('phil', default='foo', parser=bool)
else:
with pytest.raises(ValueError) as excinfo:
config('phil', default='foo', parser=bool)
assert (
str(excinfo.value) == 'ValueError: "foo" is not a valid bool value\n'
'namespace=None key=phil requires a default value parseable by everett.manager.parse_bool'
)
def build_config(cls, new_config=None):
"""Build ConfigManager using environment and overrides."""
new_config = new_config or {}
config_manager = ConfigManager(
environments=[ConfigDictEnv(new_config),
ConfigOSEnv()])
return config_manager
def pytest_runtest_setup():
# Make sure we set up logging to sane default values.
setup_logging(
ConfigManager.from_dict({
'HOST_ID': '',
'LOGGING_LEVEL': 'DEBUG'
}))
def test_parse_class_config():
config = ConfigManager.from_dict({
'foo_cls': 'hashlib.doesnotexist',
'bar_cls': 'doesnotexist.class',
})
with pytest.raises(InvalidValueError) as exc_info:
config('foo_cls', parser=parse_class)
assert (
str(exc_info.value) ==
'ValueError: "doesnotexist" is not a valid member of hashlib\n'
'namespace=None key=foo_cls requires a value parseable by everett.manager.parse_class'
)
with pytest.raises(InvalidValueError) as exc_info:
config('bar_cls', parser=parse_class)
assert (
str(exc_info.value) in
[
# Python 3
'ImportError: No module named \'doesnotexist\'\n'
'namespace=None key=bar_cls requires a value parseable by everett.manager.parse_class',
# Python 3.6
'ModuleNotFoundError: No module named \'doesnotexist\'\n'
'namespace=None key=bar_cls requires a value parseable by everett.manager.parse_class'
]
)
def test_is_thunderbird_seamonkey(self, product):
raw_crash = {
'ProductName': product
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_thunderbird_seamonkey', 100)
def main(args, config=None):
if config is None:
config = ConfigManager(
environments=[
# Pull configuration from env file specified as JANSKY_ENV
ConfigEnvFileEnv([os.environ.get('JANSKY_ENV')]),
# Pull configuration from environment variables
ConfigOSEnv()
],
doc=('For configuration help, see '
'https://jansky.readthedocs.io/en/latest/configuration.html'))
app_config = AppConfig(config)
# Set a Sentry client if we're so configured
set_sentry_client(app_config('secret_sentry_dsn'), app_config('basedir'))
# Set up logging and sentry first, so we have something to log to. Then
# build and log everything else.
setup_logging(app_config)
# Log application configuration
log_config(logger, app_config)
# Set up Sentry exception logger if we're so configured
setup_sentry_logging()
# Set up metrics
setup_metrics(app_config('metrics_class'), config, logger)
# FIXME(willkg): run processor
print('Nothing to do, yet.')
def test_tree_inferred_namespace(self):
"""Test get_runtime_config can pull namespace from config."""
config = ConfigManager.from_dict({})
class ComponentB(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("foo", parser=int, default="2")
required_config.add_option("bar", parser=int, default="1")
def __init__(self, config):
self.config = config.with_options(self)
class ComponentA(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("baz", default="abc")
def __init__(self, config):
self.config = config.with_options(self)
self.comp = ComponentB(config.with_namespace("boff"))
def get_runtime_config(self, namespace=None):
yield from super().get_runtime_config(namespace)
# Namespace here is inferred from self.comp.config which is a
# NamespacedConfig.
yield from self.comp.get_runtime_config()
comp = ComponentA(config)
assert list(comp.get_runtime_config()) == [
([], "baz", "abc", Option(key="baz", default="abc")),
(["boff"], "foo", "2", Option(key="foo", parser=int, default="2")),
(["boff"], "bar", "1", Option(key="bar", parser=int, default="1")),
]
def test_is_thunderbird_seamonkey(self, product):
raw_crash = {
'ProductName': product
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_thunderbird_seamonkey', 100)
def test_is_firefox(self, randommock):
with randommock(0.09):
raw_crash = {
'ProductName': 'Firefox',
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_firefox_desktop', 10)
with randommock(0.9):
raw_crash = {
'ProductName': 'Firefox',
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (DEFER, 'is_firefox_desktop', 10)
def test_is_fennec(self):
raw_crash = {
'ProductName': 'Fennec'
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_fennec', 100)
def test_config():
config = ConfigManager([])
# Don't raise an error and no default yields NO_VALUE
assert config('DOESNOTEXISTNOWAY', raise_error=False) is NO_VALUE
# Defaults to raising an error
with pytest.raises(ConfigurationMissingError) as exc_info:
config('DOESNOTEXISTNOWAY')
assert (
str(exc_info.value) ==
'namespace=None key=DOESNOTEXISTNOWAY requires a value parseable by str'
)
# Raises an error if raise_error is True
with pytest.raises(ConfigurationMissingError) as exc_info:
config('DOESNOTEXISTNOWAY', raise_error=True)
assert (
str(exc_info.value) ==
'namespace=None key=DOESNOTEXISTNOWAY requires a value parseable by str'
)
# With a default, returns the default
assert config('DOESNOTEXISTNOWAY', default='ohreally') == 'ohreally'
# Test doc
with pytest.raises(ConfigurationMissingError) as exc_info:
config('DOESNOTEXISTNOWAY', doc='Nothing to see here.')
assert (
str(exc_info.value) ==
'namespace=None key=DOESNOTEXISTNOWAY requires a value parseable by str\n'
'Nothing to see here.')
def test_get_namespace():
config = ConfigManager.from_dict({
'FOO': 'abc',
'FOO_BAR': 'abc',
'FOO_BAR_BAZ': 'abc',
})
assert config.get_namespace() == []
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option(
'foo',
parser=int
)
def __init__(self, config):
self.config = config.with_options(self)
def my_namespace_is(self):
return self.config.get_namespace()
comp = SomeComponent(config)
assert comp.my_namespace_is() == []
comp = SomeComponent(config.with_namespace('foo'))
assert comp.my_namespace_is() == ['foo']
def test_raw_value():
config = ConfigManager.from_dict({"FOO_BAR": "1"})
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("foo_bar", parser=int)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config)
assert comp.config("foo_bar") == 1
assert comp.config("foo_bar", raw_value=True) == "1"
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("bar", parser=int)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config.with_namespace("foo"))
assert comp.config("bar") == 1
assert comp.config("bar", raw_value=True) == "1"
def test_tree_with_specified_namespace(self):
config = ConfigManager.from_dict({})
class ComponentB(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("foo", parser=int, default="2")
required_config.add_option("bar", parser=int, default="1")
def __init__(self, config):
self.config = config.with_options(self)
class ComponentA(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option("baz", default="abc")
def __init__(self, config):
self.config = config.with_options(self)
self.comp = ComponentB(config.with_namespace("biff"))
def get_runtime_config(self, namespace=None):
for item in super(ComponentA,
self).get_runtime_config(namespace):
yield item
# We specify the namespace here
for item in self.comp.get_runtime_config(["biff"]):
yield item
comp = ComponentA(config)
assert list(comp.get_runtime_config()) == [
([], "baz", "abc", Option(key="baz", default="abc")),
(["biff"], "foo", "2", Option(key="foo", parser=int, default="2")),
(["biff"], "bar", "1", Option(key="bar", parser=int, default="1")),
]
def test_doc():
config = ConfigManager.from_dict({
'FOO_BAR': 'bat'
})
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option(
'foo_bar',
parser=int,
doc='omg!'
)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config)
try:
# This throws an exception becase "bat" is not an int
comp.config('foo_bar')
except Exception as exc:
# We're going to lazily assert that omg! is in exc msg because if it
# is, it came from the option and that's what we want to know.
assert 'omg!' in str(exc)
def test_raw_value():
config = ConfigManager.from_dict({
'FOO_BAR': '1'
})
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option(
'foo_bar',
parser=int
)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config)
assert comp.config('foo_bar') == 1
assert comp.config('foo_bar', raw_value=True) == '1'
class SomeComponent(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('bar', parser=int)
def __init__(self, config):
self.config = config.with_options(self)
comp = SomeComponent(config.with_namespace('foo'))
assert comp.config('bar') == 1
assert comp.config('bar', raw_value=True) == '1'
def test_parse_class_config():
config = ConfigManager.from_dict({
'foo_cls': 'hashlib.doesnotexist',
'bar_cls': 'doesnotexist.class',
})
if six.PY3:
with pytest.raises(InvalidValueError) as exc_info:
config('foo_cls', parser=parse_class)
else:
with pytest.raises(ValueError) as exc_info:
config('foo_cls', parser=parse_class)
assert (
str(exc_info.value) ==
'ValueError: "doesnotexist" is not a valid member of hashlib\n'
'namespace=None key=foo_cls requires a value parseable by everett.manager.parse_class'
)
if six.PY3:
with pytest.raises(InvalidValueError) as exc_info:
config('bar_cls', parser=parse_class)
else:
with pytest.raises(ImportError) as exc_info:
config('bar_cls', parser=parse_class)
assert (str(exc_info.value) in [
# Python 2
'ImportError: No module named doesnotexist\n'
'namespace=None key=bar_cls requires a value parseable by everett.manager.parse_class',
# Python 3
'ImportError: No module named \'doesnotexist\'\n'
'namespace=None key=bar_cls requires a value parseable by everett.manager.parse_class',
# Python 3.6
'ModuleNotFoundError: No module named \'doesnotexist\'\n'
'namespace=None key=bar_cls requires a value parseable by everett.manager.parse_class'
])
def test_bad_value(self):
raw_crash = {
'ProductName': ''
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (DEFER, 'NO_MATCH', 0)
def test_tree(self):
config = ConfigManager.from_dict({})
class ComponentB(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('foo', parser=int, default='2')
required_config.add_option('bar', parser=int, default='1')
def __init__(self, config):
self.config = config.with_options(self)
class ComponentA(RequiredConfigMixin):
required_config = ConfigOptions()
required_config.add_option('baz', default='abc')
def __init__(self, config):
self.config = config.with_options(self)
self.comp = ComponentB(config.with_namespace('biff'))
def get_runtime_config(self, namespace=None):
for item in super(ComponentA,
self).get_runtime_config(namespace):
yield item
for item in self.comp.get_runtime_config(['biff']):
yield item
comp = ComponentA(config)
assert (list(comp.get_runtime_config()) == [
([], 'baz', 'abc', Option(key='baz', default='abc')),
(['biff'], 'foo', '2', Option(key='foo', parser=int, default='2')),
(['biff'], 'bar', '1', Option(key='bar', parser=int, default='1')),
])
def test_is_nightly(self, channel):
raw_crash = {
'ProductName': 'Test',
'ReleaseChannel': channel
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_nightly', 100)
def test_is_version_alpha_beta_special(self, version):
raw_crash = {
'ProductName': 'Test',
'Version': version
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'is_version_alpha_beta_special', 100)
def test_comments(self):
raw_crash = {
'ProductName': 'Test',
'Comments': 'foo bar baz'
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (ACCEPT, 'has_comments', 100)
def init_app():
config = ConfigManager.from_dict({})
app_config = AppConfig(config)
logging.basicConfig(loglevel=app_config('loglevel'))
if app_config('debug'):
logging.info('debug mode!')
def test_alternate_keys(key, alternate_keys, expected):
config = ConfigManager.from_dict({
'FOO': 'foo_abc',
'FOO_BAR': 'foo_bar_abc',
'FOO_BAR_BAZ': 'foo_bar_baz_abc',
})
assert config(key, alternate_keys=alternate_keys) == expected
def get_config():
return ConfigManager([
ConfigIniEnv([
os.environ.get("CIS_CONFIG_INI"), "~/.mozilla-cis.ini",
"/etc/mozilla-cis.ini"
]),
ConfigOSEnv()
])
def rebuild_app(self, new_config):
"""Rebuilds the app
This is helpful if you've changed configuration and need to rebuild the
app so that components pick up the new configuration.
:arg new_config: dict of configuration to build the new app with
"""
self.app = get_app(ConfigManager.from_dict(new_config))
def test_invalidvalueerror():
config = ConfigManager.from_dict({
'foo_bar': 'bat'
})
with pytest.raises(InvalidValueError) as excinfo:
config('bar', namespace='foo', parser=bool)
assert excinfo.value.namespace == 'foo'
assert excinfo.value.key == 'bar'
assert excinfo.value.parser == bool
def test_hangid(self):
raw_crash = {
'ProductName': 'FireSquid',
'Version': '99',
'ProcessType': 'browser',
'HangID': 'xyz'
}
throttler = Throttler(ConfigManager.from_dict({}))
assert throttler.throttle(raw_crash) == (REJECT, 'has_hangid_and_browser', None)
def test_productname_no_unsupported_products(self):
"""Verify productname rule doesn't do anything if using ALL_PRODUCTS"""
throttler = Throttler(ConfigManager.from_dict({
'PRODUCTS': 'antenna.throttler.ALL_PRODUCTS'
}))
raw_crash = {
'ProductName': 'testproduct'
}
# This is an unsupported product, but it's not accepted for processing
# by any of the rules, so it gets caught up by the last rule
assert throttler.throttle(raw_crash) == (ACCEPT, 'accept_everything', 100)
def test_alternate_keys_with_namespace(key, alternate_keys, expected):
config = ConfigManager.from_dict({
'COMMON_FOO': 'common_foo_abc',
'FOO': 'foo_abc',
'FOO_BAR': 'foo_bar_abc',
'FOO_BAR_BAZ': 'foo_bar_baz_abc',
})
config = config.with_namespace('FOO')
assert config(key, alternate_keys=alternate_keys) == expected
def test_ListOf_error():
config = ConfigManager.from_dict({
'bools': 't,f,badbool'
})
with pytest.raises(InvalidValueError) as exc_info:
config('bools', parser=ListOf(bool))
assert (
str(exc_info.value) ==
'ValueError: "badbool" is not a valid bool value\n'
'namespace=None key=bools requires a value parseable by <ListOf(bool)>'
)
def test_with_namespace():
config = ConfigManager([
ConfigDictEnv({
'FOO_BAR': 'foobaz',
'BAR': 'baz',
'BAT': 'bat',
})
])
# Verify the values first
assert config('bar', namespace=['foo']) == 'foobaz'
assert config('bar') == 'baz'
assert config('bat') == 'bat'
# Create the namespaced config
config_with_namespace = config.with_namespace('foo')
assert config_with_namespace('bar') == 'foobaz'
# Verify 'bat' is not available because it's not in the namespace
with pytest.raises(ConfigurationError):
config_with_namespace('bat')
def test_raw_value():
config = ConfigManager.from_dict({
'FOO_BAR': '1'
})
assert config('FOO_BAR', parser=int) == 1
assert config('FOO_BAR', parser=int, raw_value=True) == '1'
assert str(config('NOEXIST', parser=int, raise_error=False)) == 'NO_VALUE'
config = config.with_namespace('FOO')
assert config('BAR', parser=int) == 1
assert config('BAR', parser=int, raw_value=True) == '1'
def test_load_files(self, client, tmpdir):
"""Verify we can rebuild the crash from the fs"""
crash_id = 'de1bb258-cbbf-4589-a673-34f800160918'
data, headers = multipart_encode({
'uuid': crash_id,
'ProductName': 'Test',
'Version': '1.0',
'upload_file_minidump': ('fakecrash.dump', io.BytesIO(b'abcd1234'))
})
# Rebuild the app the test client is using with relevant configuration.
client.rebuild_app({
'BASEDIR': str(tmpdir),
'THROTTLE_RULES': 'antenna.throttler.accept_all',
'CRASHSTORAGE_CLASS': 'antenna.ext.fs.crashstorage.FSCrashStorage',
'CRASHSTORAGE_FS_ROOT': str(tmpdir.join('antenna_crashes')),
})
result = client.simulate_post(
'/submit',
headers=headers,
body=data
)
client.join_app()
assert result.status_code == 200
config = ConfigManager.from_dict({
'FS_ROOT': str(tmpdir.join('antenna_crashes')),
})
fscrashstore = FSCrashStorage(config)
raw_crash, dumps = fscrashstore.load_raw_crash(crash_id)
assert (
raw_crash ==
{
'uuid': crash_id,
'ProductName': 'Test',
'Version': '1.0',
'dump_checksums': {'upload_file_minidump': 'e19d5cd5af0378da05f63f891c7467af'},
'legacy_processing': 0,
'throttle_rate': 100,
'submitted_timestamp': '2011-09-06T00:00:00+00:00',
'timestamp': 1315267200.0,
'type_tag': 'bp',
}
)
assert dumps == {'upload_file_minidump': b'abcd1234'}
def test_productname_reject(self, caplogpp, productname, expected):
"""Verify productname rule blocks unsupported products"""
with caplogpp.at_level(logging.INFO, logger='antenna'):
# Need a throttler with the default configuration which includes supported
# products
throttler = Throttler(ConfigManager.from_dict({}))
raw_crash = {}
if productname is not None:
raw_crash['ProductName'] = productname
assert throttler.throttle(raw_crash) == expected
assert caplogpp.record_tuples == [
('antenna.throttler', logging.INFO, 'ProductName rejected: %r' % productname)
]
def test_get_namespace():
config = ConfigManager.from_dict({
'FOO': 'abc',
'FOO_BAR': 'abc',
'FOO_BAR_BAZ': 'abc',
})
assert config.get_namespace() == []
ns_foo_config = config.with_namespace('foo')
assert ns_foo_config.get_namespace() == ['foo']
ns_foo_bar_config = ns_foo_config.with_namespace('bar')
assert ns_foo_bar_config.get_namespace() == ['foo', 'bar']
def test_productname_fakeaccept(self, caplogpp):
# This product isn't in the list and it's B2G which is the special case
with caplogpp.at_level(logging.INFO, logger='antenna'):
# Need a throttler with the default configuration which includes supported
# products
throttler = Throttler(ConfigManager.from_dict({}))
raw_crash = {
'ProductName': 'b2g'
}
assert throttler.throttle(raw_crash) == (FAKEACCEPT, 'b2g', 100)
assert caplogpp.record_tuples == [
('antenna.throttler', logging.INFO, 'ProductName B2G: fake accept')
]
