def test_write_with_imported_module_with_regex(self): required_config = Namespace() required_config.add_option( 'identifier', doc='just an identifier re', default=r'[a-zA-Z][a-zA-Z0-9]*', from_string_converter=re.compile ) cm = ConfigurationManager( required_config, values_source_list=[], ) cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # just an identifier re identifier = "[a-zA-Z][a-zA-Z0-9]*" """ self.assertEqual(generated_python_module_text, expected)
def get_es_conn(): """Return an Elasticsearch ConnectionContext.""" cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() return ConnectionContext(config)
def test_process_over_255_chars(self, mocked_subprocess_module): cm = ConfigurationManager(definition_source=( MozillaProcessorAlgorithm2015.get_required_config(), ), values_source_list=[]) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' mocked_subprocess_handle = ( mocked_subprocess_module.Popen.return_value) mocked_subprocess_handle.wait.return_value = 0 mocked_subprocess_handle.stdout.read.return_value = ( stackwalker_output_str) p = MozillaProcessorAlgorithm2015(config) raw_crash = DotDict({ "uuid": "00000000-0000-0000-0000-000002140504", "CrashTime": "1336519554", "SecondsSinceLastCrash": "86985", "PluginHang": "1", "ProductName": "Firefox", "Version": "19", "BuildID": "20121031" }) raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"} processed_crash = p.process_crash(raw_crash, raw_dumps, DotDict()) assert processed_crash.success expected = ( 'dwight; MozillaProcessorAlgorithm2015; SigTrunc: signature truncated due to length' ) assert processed_crash.processor_notes == expected assert processed_crash.signature.startswith('shutdownhang') assert len(processed_crash.signature) == 255
def test_Processor2015_init(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{'rule_sets': rule_set_02_str}], ) config = cm.get_config() config.logger = Mock() p = Processor2015(config) ok_(isinstance(p.rule_system, DotDict)) eq_(len(p.rule_system), 2) ok_('ruleset01' in p.rule_system) print p.rule_system.ruleset01 ok_(isinstance(p.rule_system.ruleset01, TransformRuleSystem)) trs = p.rule_system.ruleset01 eq_(trs.act, trs.apply_all_rules) eq_(len(trs.rules), 2) ok_(isinstance(trs.rules[0], BitguardClassifier)) ok_(isinstance(trs.rules[1], OutOfDateClassifier)) ok_('ruleset02' in p.rule_system) ok_(isinstance(p.rule_system.ruleset02, TransformRuleSystem)) trs = p.rule_system.ruleset02 eq_(trs.act, trs.apply_until_action_succeeds) eq_(len(trs.rules), 2) ok_(isinstance(trs.rules[0], SetWindowPos)) ok_(isinstance(trs.rules[1], UpdateWindowAttributes))
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, [{ 'kls_list': ('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha'), 'Alpha_00.a': 21, 'Beta_01.b': 38, }]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config.Alpha_00.a, 21) self.assertEqual(config.Beta_01.b, 38)
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option('kls_list', default='configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha', from_string_converter= converters.classes_in_namespaces_converter( 'kls%d', 'kls', instantiate_classes=True)) cm = ConfigurationManager( n, [{'kls_list':'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Beta, ' 'configman.tests.test_converters.Beta, ' 'configman.tests.test_converters.Alpha'}]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertTrue('kls_instance' in config[x]) self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_convert_raw_crash_to_processed_crash_no_rules(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{ 'rule_sets': '[]' }], ) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' p = Processor2015(config) raw_crash = DotDict() raw_dumps = {} with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow: faked_utcnow.return_value = '2015-01-01T00:00:00' processed_crash = p.convert_raw_crash_to_processed_crash( raw_crash, raw_dumps) ok_(processed_crash.success) eq_(processed_crash.started_datetime, '2015-01-01T00:00:00') eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00') eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00') eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00') eq_(processed_crash.processor_notes, 'dwight; Processor2015')
def test_convert_raw_crash_to_processed_crash_no_rules(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{'rule_sets': '[]'}], ) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' p = Processor2015(config) raw_crash = DotDict() raw_dumps = {} with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow: faked_utcnow.return_value = '2015-01-01T00:00:00' processed_crash = p.convert_raw_crash_to_processed_crash( raw_crash, raw_dumps ) ok_(processed_crash.success) eq_(processed_crash.started_datetime, '2015-01-01T00:00:00') eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00') eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00') eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00') eq_(processed_crash.processor_notes, 'dwight; Processor2015')
def test_with_transactional_resource(self): @ctm.with_transactional_resource( 'crontabber.connection_factory.ConnectionFactory', 'database' ) class Alpha(BaseCronApp): pass self.assertTrue ok_(hasattr(Alpha, "required_config")) alpha_required = Alpha.get_required_config() ok_(isinstance(alpha_required, Namespace)) ok_('database' in alpha_required) ok_('database_class' in alpha_required.database) ok_( 'database_transaction_executor_class' in alpha_required.database ) cm = ConfigurationManager( definition_source=[Alpha.get_required_config(), ], values_source_list=[environment], argv_source=[], ) config = cm.get_config() a = Alpha(config, mock.Mock()) ok_(hasattr(a, 'database_connection_factory')) ok_(isinstance(a.database_connection_factory, ConnectionFactory)) ok_(hasattr(a, 'database_transaction_executor')) ok_(isinstance(a.database_transaction_executor, TransactionExecutor))
def test_basic_05_argparse_overrides_when_appropriate(self): option_definitions = self.setup_configman_namespace() other_value_source = { "gamma": [38, 28, 18, 8] } cm = ConfigurationManager( definition_source=option_definitions, values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", "--admin.strict", '--gamma="8 18 28 38"', ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": True, "admin.expose_secrets": True } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_5(self): n = Namespace() n.add_option( 'kls_list', default=( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha' ), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d' ) ) cm = ConfigurationManager( n, [{ 'kls_list': ( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha' ), 'Alpha_00.a': 21, 'Beta_01.b': 38, }] ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for i, (a_class_name, a_class, ns_name) in \ enumerate(config.kls_list.class_list): self.assertTrue(isinstance(a_class_name, str)) self.assertEqual(a_class_name, a_class.__name__) self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_basic_02_change_all(self): option_definitions = self.setup_configman_namespace() cm = ConfigurationManager( definition_source=option_definitions, values_source_list=[command_line], argv_source=[ "16", "-b=THE SECOND", '--gamma="88 99 111 333"', "--delta", ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 16, "beta": 'THE SECOND', "gamma": [88, 99, 111, 333], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": False } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_basic_03_with_some_admin(self): option_definitions = self.setup_configman_namespace() cm = ConfigurationManager( definition_source=option_definitions, values_source_list=[command_line], argv_source=[ "0", "--admin.expose_secrets", '--gamma="-1 -2 -3 -4 -5 -6"', "--delta", "--admin.strict", ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [-1, -2, -3, -4, -5, -6], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": True, "admin.expose_secrets": True } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_write_skip_aggregations(self): required_config = Namespace() required_config.add_option( 'minimal_version_for_understanding_refusal', doc='ignore the Thottleable protocol', default={'Firefox': '3.5.4'}, ) required_config.add_aggregation('an_agg', lambda x, y, z: 'I refuse') cm = ConfigurationManager( required_config, values_source_list=[], ) config = cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # ignore the Thottleable protocol minimal_version_for_understanding_refusal = { "Firefox": "3.5.4" } """ self.assertEqual(generated_python_module_text, expected)
def test_as_overlay(self): rc = Namespace() rc.add_option('a', default=23) rc.add_option('b', default='this is b') rc.namespace('n') rc.n.add_option('x', default=datetime(1999, 12, 31, 11, 59)) rc.n.add_option('y', default=timedelta(3)) rc.n.add_option('z', default=date(1650, 10, 2)) rc.dynamic_load = None cm = ConfigurationManager( [rc], values_source_list=[ 'configman.tests.values_for_module_tests_2', 'configman.tests.values_for_module_tests_3', ]) config = cm.get_config() self.assertEqual(config.a, 99) self.assertEqual(config.b, 'now is the time') self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(config.n.y, timedelta(1)) self.assertEqual(config.n.z, date(1960, 5, 4)) from configman.tests.values_for_module_tests_3 import Alpha self.assertEqual(config.dynamic_load, Alpha) self.assertEqual(config.host, 'localhost') self.assertEqual(config.port, 5432)
def get_config(cls, values_source=None): """Return config based on the required config of the cls. This uses environment configuration and configuration defaults and lets you override that by passing in a ``values_source``. :arg cls: a configman-enhanced class :arg values_source: dict of configurable overrides :returns: a configman config object """ values_source = values_source or {} conf = cls.get_required_config() conf.add_option("logger", default=mock.Mock()) conf.add_option("metrics", default=mock.Mock()) cm = ConfigurationManager( [conf], app_name="testapp", app_version="1.0", app_description="", values_source_list=[environment, values_source], argv_source=[], ) return cm.get_config()
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( 'kls_list', default=( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha' ), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d' ) ) cm = ConfigurationManager( n, [{ 'kls_list': ( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha' ), 'Alpha_00.a': 21, 'Beta_01.b': 38, }] ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config.Alpha_00.a, 21) self.assertEqual(config.Beta_01.b, 38)
def get_tuned_config(self, sources, extra_values=None): if not isinstance(sources, (list, tuple)): sources = [sources] mock_logging = mock.Mock() config_definitions = [] for source in sources: conf = source.get_required_config() conf.add_option('logger', default=mock_logging) config_definitions.append(conf) values_source = DEFAULT_VALUES.copy() values_source.update({'logger': mock_logging}) if extra_values: values_source.update(extra_values) config_manager = ConfigurationManager( config_definitions, app_name='testapp', app_version='1.0', app_description='Elasticsearch integration tests', values_source_list=[environment, values_source], argv_source=[], ) return config_manager.get_config()
def test_basic_05_argparse_overrides_when_appropriate(self): option_definitions = self.setup_configman_namespace() other_value_source = {"gamma": [38, 28, 18, 8]} cm = ConfigurationManager( definition_source=option_definitions, values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", "--admin.strict", '--gamma="8 18 28 38"', ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": True, "admin.expose_secrets": True } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_5(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, [{ 'kls_list': ('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha'), 'Alpha_00.a': 21, 'Beta_01.b': 38, }]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for i, (a_class_name, a_class, ns_name) in (enumerate(config.kls_list.class_list)): self.assertTrue(isinstance(a_class_name, str)) self.assertEqual(a_class_name, a_class.__name__) self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_with_transactional_resource(self): @ctm.with_transactional_resource( 'socorro.external.postgresql.connection_context.ConnectionContext', 'database' ) class Alpha(BaseCronApp): pass self.assertTrue self.assertTrue(hasattr(Alpha, "required_config")) alpha_required = Alpha.get_required_config() self.assertTrue(isinstance(alpha_required, Namespace)) self.assertTrue('database' in alpha_required) self.assertTrue('database_class' in alpha_required.database) self.assertTrue( 'database_transaction_executor_class' in alpha_required.database ) cm = ConfigurationManager( definition_source=[Alpha.get_required_config(), ], values_source_list=[], argv_source=[], ) config = cm.get_config() a = Alpha(config, mock.Mock()) self.assertTrue(hasattr(a, 'database_connection')) self.assertTrue(isinstance( a.database_connection, ConnectionContext )) self.assertTrue(hasattr(a, 'database_transaction')) self.assertTrue(isinstance( a.database_transaction, TransactionExecutor ))
def test_process_crash_existing_processed_crash(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{ 'rule_sets': '[]' }], ) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' p = Processor2015(config) raw_crash = DotDict() raw_dumps = {} processed_crash = DotDict() processed_crash.processor_notes = "we've been here before; yep" processed_crash.started_datetime = '2014-01-01T00:00:00' with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow: faked_utcnow.return_value = '2015-01-01T00:00:00' processed_crash = p.process_crash(raw_crash, raw_dumps, processed_crash) assert processed_crash.success assert processed_crash.started_datetime == '2015-01-01T00:00:00' assert processed_crash.startedDateTime == '2015-01-01T00:00:00' assert processed_crash.completed_datetime == '2015-01-01T00:00:00' assert processed_crash.completeddatetime == '2015-01-01T00:00:00' expected = ( "dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; we've been here " "before; yep") assert processed_crash.processor_notes == expected
def test_process_crash_existing_processed_crash(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{'rule_sets': '[]'}], ) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' p = Processor2015(config) raw_crash = DotDict() raw_dumps = {} processed_crash = DotDict() processed_crash.processor_notes = "we've been here before; yep" processed_crash.started_datetime = '2014-01-01T00:00:00' with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow: faked_utcnow.return_value = '2015-01-01T00:00:00' processed_crash = p.process_crash( raw_crash, raw_dumps, processed_crash ) ok_(processed_crash.success) eq_(processed_crash.started_datetime, '2015-01-01T00:00:00') eq_(processed_crash.startedDateTime, '2015-01-01T00:00:00') eq_(processed_crash.completed_datetime, '2015-01-01T00:00:00') eq_(processed_crash.completeddatetime, '2015-01-01T00:00:00') eq_( processed_crash.processor_notes, "dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; " "we've been here before; yep" )
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( "kls_list", default="configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Alpha", from_string_converter=converters.classes_in_namespaces_converter("kls%d", "kls", instantiate_classes=True), ) cm = ConfigurationManager( n, [ { "kls_list": "configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Beta, " "configman.tests.test_converters.Beta, " "configman.tests.test_converters.Alpha" } ], ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertTrue("kls_instance" in config[x]) self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_Processor2015_init(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[{ 'rule_sets': rule_set_02_str }], ) config = cm.get_config() config.logger = Mock() p = Processor2015(config) ok_(isinstance(p.rule_system, DotDict)) eq_(len(p.rule_system), 2) ok_('ruleset01' in p.rule_system) ok_(isinstance(p.rule_system.ruleset01, TransformRuleSystem)) trs = p.rule_system.ruleset01 eq_(trs.act, trs.apply_all_rules) eq_(len(trs.rules), 2) ok_(isinstance(trs.rules[0], BitguardClassifier)) ok_(isinstance(trs.rules[1], OutOfDateClassifier)) ok_('ruleset02' in p.rule_system) ok_(isinstance(p.rule_system.ruleset02, TransformRuleSystem)) trs = p.rule_system.ruleset02 eq_(trs.act, trs.apply_until_action_succeeds) eq_(len(trs.rules), 2) ok_(isinstance(trs.rules[0], SetWindowPos)) ok_(isinstance(trs.rules[1], UpdateWindowAttributes))
def get_s3_context(): """Return an S3ConnectionContext.""" cm = ConfigurationManager( S3Connection.get_required_config(), values_source_list=[environment] ) config = cm.get_config() return S3Connection(config)
def get_config_context(self, es_index=None): mock_logging = mock.Mock() storage_config = \ crashstorage.ElasticSearchCrashStorage.get_required_config() middleware_config = MiddlewareApp.get_required_config() middleware_config.add_option('logger', default=mock_logging) values_source = { 'logger': mock_logging, 'resource.elasticsearch.elasticsearch_default_index': 'socorro_integration_test', 'resource.elasticsearch.elasticsearch_index': 'socorro_integration_test_reports', 'resource.elasticsearch.backoff_delays': [1], 'resource.elasticsearch.elasticsearch_timeout': 10, } if es_index: values_source['resource.elasticsearch.elasticsearch_index'] = es_index config_manager = ConfigurationManager( [storage_config, middleware_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[os.environ, values_source], argv_source=[], ) return config_manager.get_config()
def main(app_object=None): if isinstance(app_object, basestring): app_object = class_converter(app_object) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.admin = admin = Namespace() admin.add_option('application', doc='the fully qualified module or class of the ' 'application', default=app_object, from_string_converter=class_converter ) app_name = getattr(app_object, 'app_name', 'unknown') app_version = getattr(app_object, 'app_version', '0.0') app_description = getattr(app_object, 'app_description', 'no idea') # create an iterable collection of value sources # the order is important as these will supply values for the sources # defined in the_definition_source. The values will be overlain in turn. # First the os.environ values will be applied. Then any values from an ini # file parsed by getopt. Finally any values supplied on the command line # will be applied. value_sources = (ConfigFileFutureProxy, # alias for allowing the user # to specify a config file on # the command line environment, # alias for os.environ command_line) # alias for getopt # set up the manager with the definitions and values # it isn't necessary to provide the app_name because the # app_object passed in or loaded by the ConfigurationManager will alredy # have that information. config_manager = ConfigurationManager(app_definition, value_sources, app_name=app_name, app_version=app_version, app_description=app_description, ) config = config_manager.get_config() app_object = config.admin.application if isinstance(app_object, type): # invocation of the app if the app_object was a class instance = app_object(config) instance.main() elif inspect.ismodule(app_object): # invocation of the app if the app_object was a module app_object.main(config) elif inspect.isfunction(app_object): # invocation of the app if the app_object was a function app_object(config)
def main(app_object=None): if isinstance(app_object, basestring): app_object = class_converter(app_object) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.admin = admin = Namespace() admin.add_option('application', doc='the fully qualified module or class of the ' 'application', default=app_object, from_string_converter=class_converter) app_name = getattr(app_object, 'app_name', 'unknown') app_version = getattr(app_object, 'app_version', '0.0') app_description = getattr(app_object, 'app_description', 'no idea') # create an iterable collection of value sources # the order is important as these will supply values for the sources # defined in the_definition_source. The values will be overlain in turn. # First the os.environ values will be applied. Then any values from an ini # file parsed by getopt. Finally any values supplied on the command line # will be applied. value_sources = ( ConfigFileFutureProxy, # alias for allowing the user # to specify a config file on # the command line environment, # alias for os.environ command_line) # alias for getopt # set up the manager with the definitions and values # it isn't necessary to provide the app_name because the # app_object passed in or loaded by the ConfigurationManager will alredy # have that information. config_manager = ConfigurationManager( app_definition, value_sources, app_name=app_name, app_version=app_version, app_description=app_description, ) config = config_manager.get_config() app_object = config.admin.application if isinstance(app_object, type): # invocation of the app if the app_object was a class instance = app_object(config) instance.main() elif inspect.ismodule(app_object): # invocation of the app if the app_object was a module app_object.main(config) elif inspect.isfunction(app_object): # invocation of the app if the app_object was a function app_object(config)
def get_conn(): # Create a configuration manager that will only check the environment for # configuration and not command line parameters cm = ConfigurationManager(ConnectionContext.get_required_config(), values_source_list=[environment]) config = cm.get_config() return ConnectionContext(config)
def test_setup_definitions_2(self): d = {'cls': MooseBase} cm = ConfigurationManager(d, values_source_list=[]) c = cm.get_config() self.assertTrue( cm.option_definitions.cls.from_string_converter is class_converter) self.assertTrue(c.cls is MooseBase)
def test_Processor2015_init(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[], ) config = cm.get_config() config.logger = Mock() Processor2015(config)
def test_setup_definitions_2(self): d = {'cls': MooseBase} cm = ConfigurationManager(d, values_source_list=[]) c = cm.get_config() self.assertTrue( cm.option_definitions.cls.from_string_converter is class_converter ) self.assertTrue(c.cls is MooseBase)
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" cm = ConfigurationManager(ConnectionContext.get_required_config(), values_source_list=[environment]) config = cm.get_config() conn = ConnectionContext(config) yield conn for index in conn.get_indices(): conn.delete_index(index)
def get_config(self): cm = ConfigurationManager( definition_source=Processor2015.get_required_config(), values_source_list=[], ) config = cm.get_config() config.database_class = Mock() config.sentry = Mock() config.processor_name = 'dwight' return config
def get_config(self): cm = ConfigurationManager( definition_source=ProcessorPipeline.get_required_config(), values_source_list=[], ) config = cm.get_config() config.database_class = mock.Mock() config.sentry = mock.Mock() config.processor_name = "dwight" return config
def get_sqs_config(): sqs_config = SQSCrashQueue.get_required_config() config_manager = ConfigurationManager( [sqs_config], app_name="test-sqs", app_description="", values_source_list=[environment], argv_source=[], ) return config_manager.get_config()
def get_conn(): # Create a configuration manager that will only check the environment for # configuration and not command line parameters cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() return ConnectionContext(config)
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() conn = ConnectionContext(config) yield conn for index in conn.get_indices(): conn.delete_index(index)
def getconfig(): cm = ConfigurationManager(opts, values_source_list=('/etc/hangprocessor.ini', os.path.expanduser('~/hangprocessor.ini'))) config = cm.get_config() # Validate the config _validate_single_dir(config.minidump_storage_path, 'minidump_storage_path') _validate_single_dir(config.processor_queue_path, 'processor_queue_path') _validate_single_executable(config.minidump_stackwalk_path, 'minidump_stackwalk_path') config.symbol_paths = _get_dir_list(config.symbol_paths, 'symbol_paths') return config
def test_write_skip_aggregations(self): required_config = Namespace() required_config.add_option( 'minimal_version_for_understanding_refusal', doc='ignore the Thottleable protocol', default={'Firefox': '3.5.4'}, ) required_config.add_aggregation( 'an_agg', lambda x, y, z: 'I refuse' ) cm = ConfigurationManager( required_config, values_source_list=[], ) cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # ignore the Thottleable protocol minimal_version_for_understanding_refusal = { "Firefox": "3.5.4" } """ self.assertEqual(generated_python_module_text, expected)
def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = {"gamma": [38, 28, 18, 8]} other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter("configman.tests.test_val_for_modules.Delta"), "messy": 34, "dd": class_converter("configman.tests.test_val_for_modules.Beta"), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def setUp(self): super(IntegrationTestMiddlewareApp, self).setUp() self.uuid = "06a0c9b5-0381-42ce-855a-ccaaa2120116" mock_logging = mock.Mock() required_config = middleware_app.MiddlewareApp.get_required_config() required_config.add_option("logger", default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name="middleware", app_description=__doc__, values_source_list=[{"logger": mock_logging}, environment], argv_source=[], ) config = config_manager.get_config() self.conn = config.database.database_class(config.database).connection() assert self.conn.get_transaction_status() == psycopg2.extensions.TRANSACTION_STATUS_IDLE
def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = { "gamma": [38, 28, 18, 8] } other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter ) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter( "configman.tests.test_val_for_modules.Delta" ), "messy": 34, "dd": class_converter( "configman.tests.test_val_for_modules.Beta" ), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_3(self): n = Namespace() n.add_option('kls_list', default='configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha', from_string_converter= converters.classes_in_namespaces_converter('kls%d')) cm = ConfigurationManager(n, argv_source=[]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config[x].cls, Alpha) self.assertTrue('cls_instance' not in config[x])
def test_classes_in_namespaces_converter_3(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, argv_source=[]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3) self.assertTrue('Foo_00' in config) self.assertTrue('Foo_01' in config) self.assertTrue('Foo_02' in config)
def test_process_over_255_chars(self, mocked_subprocess_module): cm = ConfigurationManager( definition_source=( MozillaProcessorAlgorithm2015.get_required_config(), ), values_source_list=[] ) config = cm.get_config() config.logger = Mock() config.processor_name = 'dwight' mocked_subprocess_handle = ( mocked_subprocess_module.Popen.return_value ) mocked_subprocess_handle.wait.return_value = 0 mocked_subprocess_handle.stdout.read.return_value = ( stackwalker_output_str ) p = MozillaProcessorAlgorithm2015(config) raw_crash = DotDict({ "uuid": "00000000-0000-0000-0000-000002140504", "CrashTime": "1336519554", "SecondsSinceLastCrash": "86985", "PluginHang": "1", "ProductName": "Firefox", "Version": "19", "BuildID": "20121031" }) raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"} processed_crash = p.process_crash( raw_crash, raw_dumps, DotDict() ) ok_(processed_crash.success) eq_(processed_crash.processor_notes, 'dwight; MozillaProcessorAlgorithm2015; ' 'SignatureTool: signature truncated due to length; ' 'SignatureTool: signature truncated due to length') ok_(processed_crash.signature.startswith('shutdownhang')) eq_(len(processed_crash.signature), 255)
def test_as_overlay(self): rc = Namespace() rc.add_option( 'a', default=23 ) rc.add_option( 'b', default='this is b' ) rc.namespace('n') rc.n.add_option( 'x', default=datetime(1999, 12, 31, 11, 59) ) rc.n.add_option( 'y', default=timedelta(3) ) rc.n.add_option( 'z', default=date(1650, 10, 2) ) rc.dynamic_load = None cm = ConfigurationManager( [rc], values_source_list=[ 'configman.tests.values_for_module_tests_2', 'configman.tests.values_for_module_tests_3', ] ) config = cm.get_config() self.assertEqual(config.a, 99) self.assertEqual(config.b, 'now is the time') self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(config.n.y, timedelta(1)) self.assertEqual(config.n.z, date(1960, 5, 4)) from configman.tests.values_for_module_tests_3 import Alpha self.assertEqual(config.dynamic_load, Alpha) self.assertEqual(config.host, 'localhost') self.assertEqual(config.port, 5432)
def get_s3_context(): """Return an S3ConnectionContext.""" # The class could be anything, so get the class first cls_path = settings.SOCORRO_IMPLEMENTATIONS_CONFIG['resource']['boto']['resource_class'] module, name = cls_path.rsplit('.', 1) cls = getattr(importlib.import_module(module), name) # Now create a configuration and instantiate the class with it cm = ConfigurationManager( cls.get_required_config(), values_source_list=[ # We prefer the webapp's configuration over things in the # environment which are likely to be configman things settings.SOCORRO_IMPLEMENTATIONS_CONFIG, environment, ] ) config = cm.get_config() return cls(config)
def _get_config(self, sources, extra_values=None): self.mock_logging = mock.Mock() config_definitions = [] for source in sources: conf = source.get_required_config() conf.add_option('logger', default=self.mock_logging) config_definitions.append(conf) values_source = {'logger': self.mock_logging} config_manager = ConfigurationManager( config_definitions, app_name='testapp', app_version='1.0', app_description='', values_source_list=[values_source], argv_source=[], ) return config_manager.get_config()
def test_classes_in_namespaces_converter_3(self): n = Namespace() n.add_option( 'kls_list', default=( 'socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo' ), from_string_converter= str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d' ) ) cm = ConfigurationManager(n, argv_source=[]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3) self.assertTrue('Foo_00' in config) self.assertTrue('Foo_01' in config) self.assertTrue('Foo_02' in config)