def test_write_with_imported_module_with_regex(self):
required_config = Namespace()
required_config.add_option(
'identifier',
doc='just an identifier re',
default=r'[a-zA-Z][a-zA-Z0-9]*',
from_string_converter=re.compile
)
cm = ConfigurationManager(
required_config,
values_source_list=[],
)
cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
# just an identifier re
identifier = "[a-zA-Z][a-zA-Z0-9]*"
"""
self.assertEqual(generated_python_module_text, expected)
def test_write_with_imported_module_with_regex(self):
required_config = Namespace()
required_config.add_option(
'identifier',
doc='just an identifier re',
default=r'[a-zA-Z][a-zA-Z0-9]*',
from_string_converter=re.compile
)
cm = ConfigurationManager(
required_config,
values_source_list=[],
)
cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
# just an identifier re
identifier = "[a-zA-Z][a-zA-Z0-9]*"
"""
self.assertEqual(generated_python_module_text, expected)
def get_es_conn():
"""Return an Elasticsearch ConnectionContext."""
cm = ConfigurationManager(
ConnectionContext.get_required_config(), values_source_list=[environment]
)
config = cm.get_config()
return ConnectionContext(config)
def test_process_over_255_chars(self, mocked_subprocess_module):
cm = ConfigurationManager(definition_source=(
MozillaProcessorAlgorithm2015.get_required_config(), ),
values_source_list=[])
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
mocked_subprocess_handle = (
mocked_subprocess_module.Popen.return_value)
mocked_subprocess_handle.wait.return_value = 0
mocked_subprocess_handle.stdout.read.return_value = (
stackwalker_output_str)
p = MozillaProcessorAlgorithm2015(config)
raw_crash = DotDict({
"uuid": "00000000-0000-0000-0000-000002140504",
"CrashTime": "1336519554",
"SecondsSinceLastCrash": "86985",
"PluginHang": "1",
"ProductName": "Firefox",
"Version": "19",
"BuildID": "20121031"
})
raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"}
processed_crash = p.process_crash(raw_crash, raw_dumps, DotDict())
assert processed_crash.success
expected = (
'dwight; MozillaProcessorAlgorithm2015; SigTrunc: signature truncated due to length'
)
assert processed_crash.processor_notes == expected
assert processed_crash.signature.startswith('shutdownhang')
assert len(processed_crash.signature) == 255
def test_Processor2015_init(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{'rule_sets': rule_set_02_str}],
)
config = cm.get_config()
config.logger = Mock()
p = Processor2015(config)
ok_(isinstance(p.rule_system, DotDict))
eq_(len(p.rule_system), 2)
ok_('ruleset01' in p.rule_system)
print p.rule_system.ruleset01
ok_(isinstance(p.rule_system.ruleset01, TransformRuleSystem))
trs = p.rule_system.ruleset01
eq_(trs.act, trs.apply_all_rules)
eq_(len(trs.rules), 2)
ok_(isinstance(trs.rules[0], BitguardClassifier))
ok_(isinstance(trs.rules[1], OutOfDateClassifier))
ok_('ruleset02' in p.rule_system)
ok_(isinstance(p.rule_system.ruleset02, TransformRuleSystem))
trs = p.rule_system.ruleset02
eq_(trs.act, trs.apply_until_action_succeeds)
eq_(len(trs.rules), 2)
ok_(isinstance(trs.rules[0], SetWindowPos))
ok_(isinstance(trs.rules[1], UpdateWindowAttributes))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, [{
'kls_list': ('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'),
'Alpha_00.a':
21,
'Beta_01.b':
38,
}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config.Alpha_00.a, 21)
self.assertEqual(config.Beta_01.b, 38)
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter(
'kls%d',
'kls',
instantiate_classes=True))
cm = ConfigurationManager(
n,
[{'kls_list':'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Alpha'}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue('kls_instance' in config[x])
self.assertTrue(isinstance(config[x].kls_instance,
config[x].kls))
def test_convert_raw_crash_to_processed_crash_no_rules(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{
'rule_sets': '[]'
}],
)
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
p = Processor2015(config)
raw_crash = DotDict()
raw_dumps = {}
with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
faked_utcnow.return_value = '2015-01-01T00:00:00'
processed_crash = p.convert_raw_crash_to_processed_crash(
raw_crash, raw_dumps)
ok_(processed_crash.success)
eq_(processed_crash.started_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00')
eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00')
eq_(processed_crash.processor_notes, 'dwight; Processor2015')
def test_convert_raw_crash_to_processed_crash_no_rules(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{'rule_sets': '[]'}],
)
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
p = Processor2015(config)
raw_crash = DotDict()
raw_dumps = {}
with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
faked_utcnow.return_value = '2015-01-01T00:00:00'
processed_crash = p.convert_raw_crash_to_processed_crash(
raw_crash,
raw_dumps
)
ok_(processed_crash.success)
eq_(processed_crash.started_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00')
eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00')
eq_(processed_crash.processor_notes, 'dwight; Processor2015')
def test_with_transactional_resource(self):
@ctm.with_transactional_resource(
'crontabber.connection_factory.ConnectionFactory',
'database'
)
class Alpha(BaseCronApp):
pass
self.assertTrue
ok_(hasattr(Alpha, "required_config"))
alpha_required = Alpha.get_required_config()
ok_(isinstance(alpha_required, Namespace))
ok_('database' in alpha_required)
ok_('database_class' in alpha_required.database)
ok_(
'database_transaction_executor_class' in alpha_required.database
)
cm = ConfigurationManager(
definition_source=[Alpha.get_required_config(), ],
values_source_list=[environment],
argv_source=[],
)
config = cm.get_config()
a = Alpha(config, mock.Mock())
ok_(hasattr(a, 'database_connection_factory'))
ok_(isinstance(a.database_connection_factory, ConnectionFactory))
ok_(hasattr(a, 'database_transaction_executor'))
ok_(isinstance(a.database_transaction_executor, TransactionExecutor))
def test_basic_05_argparse_overrides_when_appropriate(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {
"gamma": [38, 28, 18, 8]
}
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[other_value_source, command_line],
argv_source=[
"0",
"--admin.expose_secrets",
"--delta",
"--admin.strict",
'--gamma="8 18 28 38"',
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [8, 18, 28, 38],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": True,
"admin.expose_secrets": True
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_5(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'
),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(
n,
[{
'kls_list': (
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'
),
'Alpha_00.a': 21,
'Beta_01.b': 38,
}]
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for i, (a_class_name, a_class, ns_name) in \
enumerate(config.kls_list.class_list):
self.assertTrue(isinstance(a_class_name, str))
self.assertEqual(a_class_name, a_class.__name__)
self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_basic_02_change_all(self):
option_definitions = self.setup_configman_namespace()
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[command_line],
argv_source=[
"16",
"-b=THE SECOND",
'--gamma="88 99 111 333"',
"--delta",
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 16,
"beta": 'THE SECOND',
"gamma": [88, 99, 111, 333],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": False,
"admin.expose_secrets": False
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_basic_03_with_some_admin(self):
option_definitions = self.setup_configman_namespace()
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[command_line],
argv_source=[
"0",
"--admin.expose_secrets",
'--gamma="-1 -2 -3 -4 -5 -6"',
"--delta",
"--admin.strict",
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [-1, -2, -3, -4, -5, -6],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": True,
"admin.expose_secrets": True
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_write_skip_aggregations(self):
required_config = Namespace()
required_config.add_option(
'minimal_version_for_understanding_refusal',
doc='ignore the Thottleable protocol',
default={'Firefox': '3.5.4'},
)
required_config.add_aggregation('an_agg', lambda x, y, z: 'I refuse')
cm = ConfigurationManager(
required_config,
values_source_list=[],
)
config = cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
# ignore the Thottleable protocol
minimal_version_for_understanding_refusal = {
"Firefox": "3.5.4"
}
"""
self.assertEqual(generated_python_module_text, expected)
def test_as_overlay(self):
rc = Namespace()
rc.add_option('a', default=23)
rc.add_option('b', default='this is b')
rc.namespace('n')
rc.n.add_option('x', default=datetime(1999, 12, 31, 11, 59))
rc.n.add_option('y', default=timedelta(3))
rc.n.add_option('z', default=date(1650, 10, 2))
rc.dynamic_load = None
cm = ConfigurationManager(
[rc],
values_source_list=[
'configman.tests.values_for_module_tests_2',
'configman.tests.values_for_module_tests_3',
])
config = cm.get_config()
self.assertEqual(config.a, 99)
self.assertEqual(config.b, 'now is the time')
self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10))
self.assertEqual(config.n.y, timedelta(1))
self.assertEqual(config.n.z, date(1960, 5, 4))
from configman.tests.values_for_module_tests_3 import Alpha
self.assertEqual(config.dynamic_load, Alpha)
self.assertEqual(config.host, 'localhost')
self.assertEqual(config.port, 5432)
def get_config(cls, values_source=None):
"""Return config based on the required config of the cls.
This uses environment configuration and configuration defaults
and lets you override that by passing in a ``values_source``.
:arg cls: a configman-enhanced class
:arg values_source: dict of configurable overrides
:returns: a configman config object
"""
values_source = values_source or {}
conf = cls.get_required_config()
conf.add_option("logger", default=mock.Mock())
conf.add_option("metrics", default=mock.Mock())
cm = ConfigurationManager(
[conf],
app_name="testapp",
app_version="1.0",
app_description="",
values_source_list=[environment, values_source],
argv_source=[],
)
return cm.get_config()
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'
),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(
n,
[{
'kls_list': (
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'
),
'Alpha_00.a': 21,
'Beta_01.b': 38,
}]
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config.Alpha_00.a, 21)
self.assertEqual(config.Beta_01.b, 38)
def get_tuned_config(self, sources, extra_values=None):
if not isinstance(sources, (list, tuple)):
sources = [sources]
mock_logging = mock.Mock()
config_definitions = []
for source in sources:
conf = source.get_required_config()
conf.add_option('logger', default=mock_logging)
config_definitions.append(conf)
values_source = DEFAULT_VALUES.copy()
values_source.update({'logger': mock_logging})
if extra_values:
values_source.update(extra_values)
config_manager = ConfigurationManager(
config_definitions,
app_name='testapp',
app_version='1.0',
app_description='Elasticsearch integration tests',
values_source_list=[environment, values_source],
argv_source=[],
)
return config_manager.get_config()
def test_basic_05_argparse_overrides_when_appropriate(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {"gamma": [38, 28, 18, 8]}
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[other_value_source, command_line],
argv_source=[
"0",
"--admin.expose_secrets",
"--delta",
"--admin.strict",
'--gamma="8 18 28 38"',
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [8, 18, 28, 38],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": True,
"admin.expose_secrets": True
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_basic_03_with_some_admin(self):
option_definitions = self.setup_configman_namespace()
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[command_line],
argv_source=[
"0",
"--admin.expose_secrets",
'--gamma="-1 -2 -3 -4 -5 -6"',
"--delta",
"--admin.strict",
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [-1, -2, -3, -4, -5, -6],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": True,
"admin.expose_secrets": True
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_basic_02_change_all(self):
option_definitions = self.setup_configman_namespace()
cm = ConfigurationManager(
definition_source=option_definitions,
values_source_list=[command_line],
argv_source=[
"16",
"-b=THE SECOND",
'--gamma="88 99 111 333"',
"--delta",
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 16,
"beta": 'THE SECOND',
"gamma": [88, 99, 111, 333],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": False,
"admin.expose_secrets": False
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_5(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, [{
'kls_list': ('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'),
'Alpha_00.a':
21,
'Beta_01.b':
38,
}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for i, (a_class_name, a_class,
ns_name) in (enumerate(config.kls_list.class_list)):
self.assertTrue(isinstance(a_class_name, str))
self.assertEqual(a_class_name, a_class.__name__)
self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_with_transactional_resource(self):
@ctm.with_transactional_resource(
'socorro.external.postgresql.connection_context.ConnectionContext',
'database'
)
class Alpha(BaseCronApp):
pass
self.assertTrue
self.assertTrue(hasattr(Alpha, "required_config"))
alpha_required = Alpha.get_required_config()
self.assertTrue(isinstance(alpha_required, Namespace))
self.assertTrue('database' in alpha_required)
self.assertTrue('database_class' in alpha_required.database)
self.assertTrue(
'database_transaction_executor_class' in alpha_required.database
)
cm = ConfigurationManager(
definition_source=[Alpha.get_required_config(), ],
values_source_list=[],
argv_source=[],
)
config = cm.get_config()
a = Alpha(config, mock.Mock())
self.assertTrue(hasattr(a, 'database_connection'))
self.assertTrue(isinstance(
a.database_connection,
ConnectionContext
))
self.assertTrue(hasattr(a, 'database_transaction'))
self.assertTrue(isinstance(
a.database_transaction,
TransactionExecutor
))
def test_process_crash_existing_processed_crash(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{
'rule_sets': '[]'
}],
)
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
p = Processor2015(config)
raw_crash = DotDict()
raw_dumps = {}
processed_crash = DotDict()
processed_crash.processor_notes = "we've been here before; yep"
processed_crash.started_datetime = '2014-01-01T00:00:00'
with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
faked_utcnow.return_value = '2015-01-01T00:00:00'
processed_crash = p.process_crash(raw_crash, raw_dumps,
processed_crash)
assert processed_crash.success
assert processed_crash.started_datetime == '2015-01-01T00:00:00'
assert processed_crash.startedDateTime == '2015-01-01T00:00:00'
assert processed_crash.completed_datetime == '2015-01-01T00:00:00'
assert processed_crash.completeddatetime == '2015-01-01T00:00:00'
expected = (
"dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; we've been here "
"before; yep")
assert processed_crash.processor_notes == expected
def test_process_crash_existing_processed_crash(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{'rule_sets': '[]'}],
)
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
p = Processor2015(config)
raw_crash = DotDict()
raw_dumps = {}
processed_crash = DotDict()
processed_crash.processor_notes = "we've been here before; yep"
processed_crash.started_datetime = '2014-01-01T00:00:00'
with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
faked_utcnow.return_value = '2015-01-01T00:00:00'
processed_crash = p.process_crash(
raw_crash,
raw_dumps,
processed_crash
)
ok_(processed_crash.success)
eq_(processed_crash.started_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00')
eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00')
eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00')
eq_(
processed_crash.processor_notes,
"dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; "
"we've been here before; yep"
)
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
"kls_list",
default="configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Alpha",
from_string_converter=converters.classes_in_namespaces_converter("kls%d", "kls", instantiate_classes=True),
)
cm = ConfigurationManager(
n,
[
{
"kls_list": "configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Beta, "
"configman.tests.test_converters.Beta, "
"configman.tests.test_converters.Alpha"
}
],
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue("kls_instance" in config[x])
self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter(
'kls%d',
'kls',
instantiate_classes=True))
cm = ConfigurationManager(
n,
[{'kls_list':'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Alpha'}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue('kls_instance' in config[x])
self.assertTrue(isinstance(config[x].kls_instance,
config[x].kls))
def test_Processor2015_init(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[{
'rule_sets': rule_set_02_str
}],
)
config = cm.get_config()
config.logger = Mock()
p = Processor2015(config)
ok_(isinstance(p.rule_system, DotDict))
eq_(len(p.rule_system), 2)
ok_('ruleset01' in p.rule_system)
ok_(isinstance(p.rule_system.ruleset01, TransformRuleSystem))
trs = p.rule_system.ruleset01
eq_(trs.act, trs.apply_all_rules)
eq_(len(trs.rules), 2)
ok_(isinstance(trs.rules[0], BitguardClassifier))
ok_(isinstance(trs.rules[1], OutOfDateClassifier))
ok_('ruleset02' in p.rule_system)
ok_(isinstance(p.rule_system.ruleset02, TransformRuleSystem))
trs = p.rule_system.ruleset02
eq_(trs.act, trs.apply_until_action_succeeds)
eq_(len(trs.rules), 2)
ok_(isinstance(trs.rules[0], SetWindowPos))
ok_(isinstance(trs.rules[1], UpdateWindowAttributes))
def get_s3_context():
"""Return an S3ConnectionContext."""
cm = ConfigurationManager(
S3Connection.get_required_config(), values_source_list=[environment]
)
config = cm.get_config()
return S3Connection(config)
def get_tuned_config(self, sources, extra_values=None):
if not isinstance(sources, (list, tuple)):
sources = [sources]
mock_logging = mock.Mock()
config_definitions = []
for source in sources:
conf = source.get_required_config()
conf.add_option('logger', default=mock_logging)
config_definitions.append(conf)
values_source = DEFAULT_VALUES.copy()
values_source.update({'logger': mock_logging})
if extra_values:
values_source.update(extra_values)
config_manager = ConfigurationManager(
config_definitions,
app_name='testapp',
app_version='1.0',
app_description='Elasticsearch integration tests',
values_source_list=[environment, values_source],
argv_source=[],
)
return config_manager.get_config()
def get_config_context(self, es_index=None):
mock_logging = mock.Mock()
storage_config = \
crashstorage.ElasticSearchCrashStorage.get_required_config()
middleware_config = MiddlewareApp.get_required_config()
middleware_config.add_option('logger', default=mock_logging)
values_source = {
'logger': mock_logging,
'resource.elasticsearch.elasticsearch_default_index': 'socorro_integration_test',
'resource.elasticsearch.elasticsearch_index': 'socorro_integration_test_reports',
'resource.elasticsearch.backoff_delays': [1],
'resource.elasticsearch.elasticsearch_timeout': 10,
}
if es_index:
values_source['resource.elasticsearch.elasticsearch_index'] = es_index
config_manager = ConfigurationManager(
[storage_config, middleware_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[os.environ, values_source],
argv_source=[],
)
return config_manager.get_config()
def main(app_object=None):
if isinstance(app_object, basestring):
app_object = class_converter(app_object)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.admin = admin = Namespace()
admin.add_option('application',
doc='the fully qualified module or class of the '
'application',
default=app_object,
from_string_converter=class_converter
)
app_name = getattr(app_object, 'app_name', 'unknown')
app_version = getattr(app_object, 'app_version', '0.0')
app_description = getattr(app_object, 'app_description', 'no idea')
# create an iterable collection of value sources
# the order is important as these will supply values for the sources
# defined in the_definition_source. The values will be overlain in turn.
# First the os.environ values will be applied. Then any values from an ini
# file parsed by getopt. Finally any values supplied on the command line
# will be applied.
value_sources = (ConfigFileFutureProxy, # alias for allowing the user
# to specify a config file on
# the command line
environment, # alias for os.environ
command_line) # alias for getopt
# set up the manager with the definitions and values
# it isn't necessary to provide the app_name because the
# app_object passed in or loaded by the ConfigurationManager will alredy
# have that information.
config_manager = ConfigurationManager(app_definition,
value_sources,
app_name=app_name,
app_version=app_version,
app_description=app_description,
)
config = config_manager.get_config()
app_object = config.admin.application
if isinstance(app_object, type):
# invocation of the app if the app_object was a class
instance = app_object(config)
instance.main()
elif inspect.ismodule(app_object):
# invocation of the app if the app_object was a module
app_object.main(config)
elif inspect.isfunction(app_object):
# invocation of the app if the app_object was a function
app_object(config)
def main(app_object=None):
if isinstance(app_object, basestring):
app_object = class_converter(app_object)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.admin = admin = Namespace()
admin.add_option('application',
doc='the fully qualified module or class of the '
'application',
default=app_object,
from_string_converter=class_converter)
app_name = getattr(app_object, 'app_name', 'unknown')
app_version = getattr(app_object, 'app_version', '0.0')
app_description = getattr(app_object, 'app_description', 'no idea')
# create an iterable collection of value sources
# the order is important as these will supply values for the sources
# defined in the_definition_source. The values will be overlain in turn.
# First the os.environ values will be applied. Then any values from an ini
# file parsed by getopt. Finally any values supplied on the command line
# will be applied.
value_sources = (
ConfigFileFutureProxy, # alias for allowing the user
# to specify a config file on
# the command line
environment, # alias for os.environ
command_line) # alias for getopt
# set up the manager with the definitions and values
# it isn't necessary to provide the app_name because the
# app_object passed in or loaded by the ConfigurationManager will alredy
# have that information.
config_manager = ConfigurationManager(
app_definition,
value_sources,
app_name=app_name,
app_version=app_version,
app_description=app_description,
)
config = config_manager.get_config()
app_object = config.admin.application
if isinstance(app_object, type):
# invocation of the app if the app_object was a class
instance = app_object(config)
instance.main()
elif inspect.ismodule(app_object):
# invocation of the app if the app_object was a module
app_object.main(config)
elif inspect.isfunction(app_object):
# invocation of the app if the app_object was a function
app_object(config)
def get_conn():
# Create a configuration manager that will only check the environment for
# configuration and not command line parameters
cm = ConfigurationManager(ConnectionContext.get_required_config(),
values_source_list=[environment])
config = cm.get_config()
return ConnectionContext(config)
def test_setup_definitions_2(self):
d = {'cls': MooseBase}
cm = ConfigurationManager(d, values_source_list=[])
c = cm.get_config()
self.assertTrue(
cm.option_definitions.cls.from_string_converter is class_converter)
self.assertTrue(c.cls is MooseBase)
def test_Processor2015_init(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[],
)
config = cm.get_config()
config.logger = Mock()
Processor2015(config)
def test_setup_definitions_2(self):
d = {'cls': MooseBase}
cm = ConfigurationManager(d, values_source_list=[])
c = cm.get_config()
self.assertTrue(
cm.option_definitions.cls.from_string_converter is class_converter
)
self.assertTrue(c.cls is MooseBase)
def es_conn():
"""Create an Elasticsearch ConnectionContext and clean up indices afterwards."""
cm = ConfigurationManager(ConnectionContext.get_required_config(),
values_source_list=[environment])
config = cm.get_config()
conn = ConnectionContext(config)
yield conn
for index in conn.get_indices():
conn.delete_index(index)
def get_config(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[],
)
config = cm.get_config()
config.database_class = Mock()
config.sentry = Mock()
config.processor_name = 'dwight'
return config
def get_config(self):
cm = ConfigurationManager(
definition_source=ProcessorPipeline.get_required_config(),
values_source_list=[],
)
config = cm.get_config()
config.database_class = mock.Mock()
config.sentry = mock.Mock()
config.processor_name = "dwight"
return config
def get_sqs_config():
sqs_config = SQSCrashQueue.get_required_config()
config_manager = ConfigurationManager(
[sqs_config],
app_name="test-sqs",
app_description="",
values_source_list=[environment],
argv_source=[],
)
return config_manager.get_config()
def get_conn():
# Create a configuration manager that will only check the environment for
# configuration and not command line parameters
cm = ConfigurationManager(
ConnectionContext.get_required_config(),
values_source_list=[environment]
)
config = cm.get_config()
return ConnectionContext(config)
def get_config(self):
cm = ConfigurationManager(
definition_source=Processor2015.get_required_config(),
values_source_list=[],
)
config = cm.get_config()
config.database_class = Mock()
config.sentry = Mock()
config.processor_name = 'dwight'
return config
def es_conn():
"""Create an Elasticsearch ConnectionContext and clean up indices afterwards."""
cm = ConfigurationManager(
ConnectionContext.get_required_config(),
values_source_list=[environment]
)
config = cm.get_config()
conn = ConnectionContext(config)
yield conn
for index in conn.get_indices():
conn.delete_index(index)
def getconfig():
cm = ConfigurationManager(opts,
values_source_list=('/etc/hangprocessor.ini',
os.path.expanduser('~/hangprocessor.ini')))
config = cm.get_config()
# Validate the config
_validate_single_dir(config.minidump_storage_path, 'minidump_storage_path')
_validate_single_dir(config.processor_queue_path, 'processor_queue_path')
_validate_single_executable(config.minidump_stackwalk_path, 'minidump_stackwalk_path')
config.symbol_paths = _get_dir_list(config.symbol_paths, 'symbol_paths')
return config
def test_write_skip_aggregations(self):
required_config = Namespace()
required_config.add_option(
'minimal_version_for_understanding_refusal',
doc='ignore the Thottleable protocol',
default={'Firefox': '3.5.4'},
)
required_config.add_aggregation(
'an_agg',
lambda x, y, z: 'I refuse'
)
cm = ConfigurationManager(
required_config,
values_source_list=[],
)
cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
# ignore the Thottleable protocol
minimal_version_for_understanding_refusal = {
"Firefox": "3.5.4"
}
"""
self.assertEqual(generated_python_module_text, expected)
def test_basic_07_argparse_multilevel_class_expansion(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {"gamma": [38, 28, 18, 8]}
other_definition_source = Namespace()
other_definition_source.add_option(
"a_class",
default="configman.tests.test_val_for_modules.Alpha",
from_string_converter=class_converter)
cm = ConfigurationManager(
definition_source=[option_definitions, other_definition_source],
values_source_list=[other_value_source, command_line],
argv_source=[
"0", "--admin.expose_secrets", "--delta",
'--gamma="8 18 28 38"',
'--a_class=configman.tests.test_val_for_modules.Delta',
'--messy=34'
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha":
0,
"beta":
'the second',
"gamma": [8, 18, 28, 38],
"delta":
True,
"admin.print_conf":
None,
"admin.dump_conf":
'',
"admin.strict":
False,
"admin.expose_secrets":
True,
"a_class":
class_converter("configman.tests.test_val_for_modules.Delta"),
"messy":
34,
"dd":
class_converter("configman.tests.test_val_for_modules.Beta"),
'b':
23,
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def setUp(self):
super(IntegrationTestMiddlewareApp, self).setUp()
self.uuid = "06a0c9b5-0381-42ce-855a-ccaaa2120116"
mock_logging = mock.Mock()
required_config = middleware_app.MiddlewareApp.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="middleware",
app_description=__doc__,
values_source_list=[{"logger": mock_logging}, environment],
argv_source=[],
)
config = config_manager.get_config()
self.conn = config.database.database_class(config.database).connection()
assert self.conn.get_transaction_status() == psycopg2.extensions.TRANSACTION_STATUS_IDLE
def test_basic_07_argparse_multilevel_class_expansion(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {
"gamma": [38, 28, 18, 8]
}
other_definition_source = Namespace()
other_definition_source.add_option(
"a_class",
default="configman.tests.test_val_for_modules.Alpha",
from_string_converter=class_converter
)
cm = ConfigurationManager(
definition_source=[option_definitions, other_definition_source],
values_source_list=[other_value_source, command_line],
argv_source=[
"0",
"--admin.expose_secrets",
"--delta",
'--gamma="8 18 28 38"',
'--a_class=configman.tests.test_val_for_modules.Delta',
'--messy=34'
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [8, 18, 28, 38],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": False,
"admin.expose_secrets": True,
"a_class": class_converter(
"configman.tests.test_val_for_modules.Delta"
),
"messy": 34,
"dd": class_converter(
"configman.tests.test_val_for_modules.Beta"
),
'b': 23,
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter('kls%d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config[x].cls, Alpha)
self.assertTrue('cls_instance' not in config[x])
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter('kls%d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config[x].cls, Alpha)
self.assertTrue('cls_instance' not in config[x])
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
self.assertTrue('Foo_00' in config)
self.assertTrue('Foo_01' in config)
self.assertTrue('Foo_02' in config)
def test_process_over_255_chars(self, mocked_subprocess_module):
cm = ConfigurationManager(
definition_source=(
MozillaProcessorAlgorithm2015.get_required_config(),
),
values_source_list=[]
)
config = cm.get_config()
config.logger = Mock()
config.processor_name = 'dwight'
mocked_subprocess_handle = (
mocked_subprocess_module.Popen.return_value
)
mocked_subprocess_handle.wait.return_value = 0
mocked_subprocess_handle.stdout.read.return_value = (
stackwalker_output_str
)
p = MozillaProcessorAlgorithm2015(config)
raw_crash = DotDict({
"uuid": "00000000-0000-0000-0000-000002140504",
"CrashTime": "1336519554",
"SecondsSinceLastCrash": "86985",
"PluginHang": "1",
"ProductName": "Firefox",
"Version": "19",
"BuildID": "20121031"
})
raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"}
processed_crash = p.process_crash(
raw_crash,
raw_dumps,
DotDict()
)
ok_(processed_crash.success)
eq_(processed_crash.processor_notes,
'dwight; MozillaProcessorAlgorithm2015; '
'SignatureTool: signature truncated due to length; '
'SignatureTool: signature truncated due to length')
ok_(processed_crash.signature.startswith('shutdownhang'))
eq_(len(processed_crash.signature), 255)
def test_as_overlay(self):
rc = Namespace()
rc.add_option(
'a',
default=23
)
rc.add_option(
'b',
default='this is b'
)
rc.namespace('n')
rc.n.add_option(
'x',
default=datetime(1999, 12, 31, 11, 59)
)
rc.n.add_option(
'y',
default=timedelta(3)
)
rc.n.add_option(
'z',
default=date(1650, 10, 2)
)
rc.dynamic_load = None
cm = ConfigurationManager(
[rc],
values_source_list=[
'configman.tests.values_for_module_tests_2',
'configman.tests.values_for_module_tests_3',
]
)
config = cm.get_config()
self.assertEqual(config.a, 99)
self.assertEqual(config.b, 'now is the time')
self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10))
self.assertEqual(config.n.y, timedelta(1))
self.assertEqual(config.n.z, date(1960, 5, 4))
from configman.tests.values_for_module_tests_3 import Alpha
self.assertEqual(config.dynamic_load, Alpha)
self.assertEqual(config.host, 'localhost')
self.assertEqual(config.port, 5432)
def get_s3_context():
"""Return an S3ConnectionContext."""
# The class could be anything, so get the class first
cls_path = settings.SOCORRO_IMPLEMENTATIONS_CONFIG['resource']['boto']['resource_class']
module, name = cls_path.rsplit('.', 1)
cls = getattr(importlib.import_module(module), name)
# Now create a configuration and instantiate the class with it
cm = ConfigurationManager(
cls.get_required_config(),
values_source_list=[
# We prefer the webapp's configuration over things in the
# environment which are likely to be configman things
settings.SOCORRO_IMPLEMENTATIONS_CONFIG,
environment,
]
)
config = cm.get_config()
return cls(config)
def _get_config(self, sources, extra_values=None):
self.mock_logging = mock.Mock()
config_definitions = []
for source in sources:
conf = source.get_required_config()
conf.add_option('logger', default=self.mock_logging)
config_definitions.append(conf)
values_source = {'logger': self.mock_logging}
config_manager = ConfigurationManager(
config_definitions,
app_name='testapp',
app_version='1.0',
app_description='',
values_source_list=[values_source],
argv_source=[],
)
return config_manager.get_config()
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo'
),
from_string_converter= str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
self.assertTrue('Foo_00' in config)
self.assertTrue('Foo_01' in config)
self.assertTrue('Foo_02' in config)
