def test_for_mapping_long_doc_in_write_conf(self): n = self._some_namespaces() n = Namespace(doc='top') n.add_option( 'aaa', 'Default Value Goes In Here', 'This time the documentation string is really long. So long ' 'that we have to write it on multiple lines.', ) cm = ConfigurationManager( n, values_source_list=[], ) out = StringIO() cm.write_conf(for_mapping, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() for line in received.splitlines(): self.assertTrue(len(line) < 80, line) expected = """ # This time the documentation string is really long. So long that we have to # write it on multiple lines. (default: 'Default Value Goes In Here') aaa='Default Value Goes In Here' """.strip() self.assertEqual(received.strip(), expected)
def config_from_configman(): """Generate a configman DotDict to pass to configman components.""" definition_source = Namespace() definition_source.namespace("logging") definition_source.logging = App.required_config.logging definition_source.namespace("metricscfg") definition_source.metricscfg = App.required_config.metricscfg definition_source.namespace("elasticsearch") definition_source.elasticsearch.add_option("elasticsearch_class", default=ESConnectionContext) definition_source.namespace("queue") definition_source.add_option("crashqueue_class", default=import_string(settings.CRASHQUEUE)) definition_source.namespace("crashdata") definition_source.crashdata.add_option("crash_data_class", default=SimplifiedCrashData) definition_source.namespace("telemetrydata") definition_source.telemetrydata.add_option("telemetry_data_class", default=TelemetryCrashData) return configuration( definition_source=definition_source, values_source_list=[settings.SOCORRO_CONFIG], )
def test_polycrashstorage_processed_immutability_with_nonmutating(self): """Verifies if a crash storage says it doesn't mutate the class that " we don't do a deepcopy """ n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_namespaces': 'store1', 'store1.crashstorage_class': ('socorro.unittest.external.test_crashstorage_base' '.NonMutatingProcessedCrashCrashStorage'), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = {'foo': 'bar'} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') # We have a crashstorage that says it's not mutating, but deletes a # key so that we can verify that the code went down the right path # in the processor. assert 'foo' not in processed_crash
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, }], argv_source=[]) with config_manager.context() as config: crashstorage = CrashStorageBase( config, quit_check_callback=fake_quit_check) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.save_processed({}) assert_raises(NotImplementedError, crashstorage.get_raw_crash, 'ooid') assert_raises(NotImplementedError, crashstorage.get_raw_dump, 'ooid') assert_raises(NotImplementedError, crashstorage.get_unredacted_processed, 'ooid') assert_raises(NotImplementedError, crashstorage.remove, 'ooid') eq_(crashstorage.new_crashes(), []) crashstorage.close()
def config_from_configman(): """Generate a configman DotDict to pass to configman components.""" definition_source = Namespace() definition_source.namespace('logging') definition_source.logging = App.required_config.logging definition_source.namespace('metricscfg') definition_source.metricscfg = App.required_config.metricscfg definition_source.namespace('elasticsearch') definition_source.elasticsearch.add_option( 'elasticsearch_class', default=ESConnectionContext, ) definition_source.namespace('queue') definition_source.add_option('crashqueue_class', default=PubSubCrashQueue) definition_source.namespace('crashdata') definition_source.crashdata.add_option( 'crash_data_class', default=SimplifiedCrashData, ) definition_source.namespace('telemetrydata') definition_source.telemetrydata.add_option( 'telemetry_data_class', default=TelemetryCrashData, ) return configuration(definition_source=definition_source, values_source_list=[ settings.SOCORRO_IMPLEMENTATIONS_CONFIG, ])
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( "kls_list", default="configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Alpha", from_string_converter=converters.classes_in_namespaces_converter("kls%d", "kls", instantiate_classes=True), ) cm = ConfigurationManager( n, [ { "kls_list": "configman.tests.test_converters.Alpha, " "configman.tests.test_converters.Beta, " "configman.tests.test_converters.Beta, " "configman.tests.test_converters.Alpha" } ], ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertTrue("kls_instance" in config[x]) self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_write_skip_aggregations(self): required_config = Namespace() required_config.add_option( 'minimal_version_for_understanding_refusal', doc='ignore the Thottleable protocol', default={'Firefox': '3.5.4'}, ) required_config.add_aggregation('an_agg', lambda x, y, z: 'I refuse') cm = ConfigurationManager( required_config, values_source_list=[], ) config = cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # ignore the Thottleable protocol minimal_version_for_understanding_refusal = { "Firefox": "3.5.4" } """ self.assertEqual(generated_python_module_text, expected)
def test_write_with_imported_module_with_regex(self): required_config = Namespace() required_config.add_option('identifier', doc='just an identifier re', default=r'[a-zA-Z][a-zA-Z0-9]*', from_string_converter=re.compile) cm = ConfigurationManager( required_config, values_source_list=[], ) config = cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # just an identifier re identifier = "[a-zA-Z][a-zA-Z0-9]*" """ self.assertEqual(generated_python_module_text, expected)
def run_main(main_function, configuration_requirements=Namespace()): required_config = Namespace() required_config.add_option( 'rule_system_class', doc='the fully qualified name of the RuleSystem class', default=RuleSystem, from_string_converter=class_converter, ) required_config.update(logging_config) required_config.update(configuration_requirements) config = configuration(required_config) logging.basicConfig(level=config.logging_level, format=config.logging_format) log_config(config) rule_system = config.rule_system_class(config) loop = asyncio.get_event_loop() loop.run_until_complete(rule_system.initialize()) main_function(config, rule_system) loop.run_until_complete(rule_system.go()) try: loop.run_forever() except KeyboardInterrupt: pass
def test_write_with_imported_module_with_regex(self): required_config = Namespace() required_config.add_option( 'identifier', doc='just an identifier re', default=r'[a-zA-Z][a-zA-Z0-9]*', from_string_converter=re.compile ) cm = ConfigurationManager( required_config, values_source_list=[], ) config = cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file # just an identifier re identifier = "[a-zA-Z][a-zA-Z0-9]*" """ self.assertEqual(generated_python_module_text, expected)
def test_as_overlay(self): rc = Namespace() rc.add_option('a', default=23) rc.add_option('b', default='this is b') rc.namespace('n') rc.n.add_option('x', default=datetime(1999, 12, 31, 11, 59)) rc.n.add_option('y', default=timedelta(3)) rc.n.add_option('z', default=date(1650, 10, 2)) rc.dynamic_load = None cm = ConfigurationManager( [rc], values_source_list=[ 'configman.tests.values_for_module_tests_2', 'configman.tests.values_for_module_tests_3', ]) config = cm.get_config() self.assertEqual(config.a, 99) self.assertEqual(config.b, 'now is the time') self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(config.n.y, timedelta(1)) self.assertEqual(config.n.z, date(1960, 5, 4)) from configman.tests.values_for_module_tests_3 import Alpha self.assertEqual(config.dynamic_load, Alpha) self.assertEqual(config.host, 'localhost') self.assertEqual(config.port, 5432)
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( 'kls_list', default=( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha' ), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d' ) ) cm = ConfigurationManager( n, [{ 'kls_list': ( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha' ), 'Alpha_00.a': 21, 'Beta_01.b': 38, }] ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config.Alpha_00.a, 21) self.assertEqual(config.Beta_01.b, 38)
def main(initial_app, values_source_list=None, config_path=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) if config_path is None: default = './config' config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', default) if config_path != default: # you tried to set it, then it must be a valid directory if not os.path.isdir(config_path): raise IOError('%s is not a valid directory' % config_path) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def donttest_for_configobj_basics_3(self): n = Namespace() n.add_option("name", default='lars') n.add_option("awesome", default='lars') n.namespace('othersection') n.othersection.add_option('foo', default=23) tmp_filename = os.path.join(tempfile.gettempdir(), 'test.ini') open(tmp_filename, 'w').write(""" # comment name=Peter awesome= # comment [othersection] bad_option=bar # other comment """) try: self.assertRaises( NotAnOptionError, config_manager.ConfigurationManager, [n], [tmp_filename], ) finally: if os.path.isfile(tmp_filename): os.remove(tmp_filename)
def get_extra_as_options(input_str): if '|' not in input_str: raise JobDescriptionError('No frequency and/or time defined') metadata = input_str.split('|')[1:] if len(metadata) == 1: if ':' in metadata[0]: frequency = '1d' time_ = metadata[0] else: frequency = metadata[0] time_ = None else: frequency, time_ = metadata n = Namespace() n.add_option( 'frequency', doc='frequency', default=frequency, #from_string_converter=int exclude_from_print_conf=True, exclude_from_dump_conf=True ) n.add_option( 'time', doc='time', default=time_, exclude_from_print_conf=True, exclude_from_dump_conf=True ) return n
def test_classes_in_namespaces_converter_5(self): n = Namespace() n.add_option( 'kls_list', default=( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha' ), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d' ) ) cm = ConfigurationManager( n, [{ 'kls_list': ( 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha' ), 'Alpha_00.a': 21, 'Beta_01.b': 38, }] ) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for i, (a_class_name, a_class, ns_name) in \ enumerate(config.kls_list.class_list): self.assertTrue(isinstance(a_class_name, str)) self.assertEqual(a_class_name, a_class.__name__) self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_classes_in_namespaces_converter_5(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, [{ 'kls_list': ('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha'), 'Alpha_00.a': 21, 'Beta_01.b': 38, }]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for i, (a_class_name, a_class, ns_name) in (enumerate(config.kls_list.class_list)): self.assertTrue(isinstance(a_class_name, str)) self.assertEqual(a_class_name, a_class.__name__) self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_classes': ( 'socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage' ), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = { 'foo': DotDict({'other': 'thing'}), 'bar': SocorroDotDict({'something': 'else'}), } poly_store = config.storage(config) poly_store.save_raw_and_processed( raw_crash, dump, processed_crash, 'n' ) eq_(processed_crash['foo']['other'], 'thing') eq_(processed_crash['bar']['something'], 'else')
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Alpha'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, [{ 'kls_list': ('socorro.unittest.lib.test_converters.Alpha, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Beta, ' 'socorro.unittest.lib.test_converters.Alpha'), 'Alpha_00.a': 21, 'Beta_01.b': 38, }]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config.Alpha_00.a, 21) self.assertEqual(config.Beta_01.b, 38)
def test_polycrashstorage_processed_immutability_with_nonmutating(self): """Verifies if a crash storage says it doesn't mutate the class that " we don't do a deepcopy """ n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".NonMutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = {"foo": "bar"} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") # We have a crashstorage that says it's not mutating, but deletes a # key so that we can verify that the code went down the right path # in the processor. assert "foo" not in processed_crash
def define_config(): definition = Namespace() definition.add_option( name='devowel', default=False ) return definition
def test_classes_in_namespaces_converter_4(self): n = Namespace() n.add_option('kls_list', default='configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha', from_string_converter= converters.classes_in_namespaces_converter( 'kls%d', 'kls', instantiate_classes=True)) cm = ConfigurationManager( n, [{'kls_list':'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Beta, ' 'configman.tests.test_converters.Beta, ' 'configman.tests.test_converters.Alpha'}]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertTrue('kls_instance' in config[x]) self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_basic_usage_with_postgres_with_backoff(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, #default=TransactionExecutor, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], argv_source=[]) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) _function_calls = [] # some mutable def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) executor(mock_function) ok_(_function_calls) eq_(commit_count, 1) eq_(rollback_count, 0)
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, }] ) with config_manager.context() as config: crashstorage = CrashStorageBase( config, quit_check_callback=fake_quit_check ) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.save_processed({}) self.assertRaises(NotImplementedError, crashstorage.get_raw_crash, 'ooid') self.assertRaises(NotImplementedError, crashstorage.get_raw_dump, 'ooid') self.assertRaises(NotImplementedError, crashstorage.get_processed, 'ooid') self.assertRaises(NotImplementedError, crashstorage.remove, 'ooid') self.assertRaises(StopIteration, crashstorage.new_crashes) crashstorage.close()
def get_extra_as_options(input_str): if '|' not in input_str: raise JobDescriptionError('No frequency and/or time defined') metadata = input_str.split('|')[1:] if len(metadata) == 1: if ':' in metadata[0]: frequency = '1d' time_ = metadata[0] else: frequency = metadata[0] time_ = None else: frequency, time_ = metadata n = Namespace() n.add_option('frequency', doc='frequency', default=frequency, exclude_from_print_conf=True, exclude_from_dump_conf=True) n.add_option('time', doc='time', default=time_, exclude_from_print_conf=True, exclude_from_dump_conf=True) return n
def test_no_rollback_exception_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutor, doc='a class that will execute transactions' ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{'database_class': MockConnectionContext}], ) with config_manager.context() as config: executor = config.transaction_executor_class(config) def mock_function(connection): assert isinstance(connection, MockConnection) raise NameError('crap!') self.assertRaises(NameError, executor, mock_function) self.assertEqual(commit_count, 0) self.assertEqual(rollback_count, 0) self.assertTrue(mock_logging.errors)
def config_from_configman(): definition_source = Namespace() definition_source.namespace('logging') definition_source.logging = socorro_app.App.required_config.logging definition_source.namespace('metricscfg') definition_source.metricscfg = socorro_app.App.required_config.metricscfg definition_source.namespace('elasticsearch') definition_source.elasticsearch.add_option( 'elasticsearch_class', default=ESConnectionContext, ) definition_source.namespace('queue') definition_source.add_option( 'crashqueue_class', default=PubSubCrashQueue ) definition_source.namespace('crashdata') definition_source.crashdata.add_option( 'crash_data_class', default=socorro.external.boto.crash_data.SimplifiedCrashData, ) definition_source.namespace('telemetrydata') definition_source.telemetrydata.add_option( 'telemetry_data_class', default=socorro.external.boto.crash_data.TelemetryCrashData, ) return configuration( definition_source=definition_source, values_source_list=[ settings.SOCORRO_IMPLEMENTATIONS_CONFIG, ] )
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = mock.Mock() required_config.add_option("logger", default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{ "logger": mock_logging }], argv_source=[], ) with config_manager.context() as config: crashstorage = CrashStorageBase(config) crashstorage.save_raw_crash({}, "payload", "ooid") with pytest.raises(NotImplementedError): crashstorage.get_raw_crash("ooid") with pytest.raises(NotImplementedError): crashstorage.get_raw_dump("ooid") with pytest.raises(NotImplementedError): crashstorage.get_unredacted_processed("ooid") with pytest.raises(NotImplementedError): crashstorage.remove("ooid") crashstorage.close()
def main(initial_app, values_source_list=None, config_path=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) if config_path is None: default = './config' config_path = os.environ.get( 'DEFAULT_SOCORRO_CONFIG_PATH', default ) if config_path != default: # you tried to set it, then it must be a valid directory if not os.path.isdir(config_path): raise IOError('%s is not a valid directory' % config_path) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def _some_namespaces(self): """set up some namespaces""" n = Namespace(doc='top') n.add_option('aaa', '2011-05-04T15:10:00', 'the a', short_form='a', from_string_converter=dtu.datetime_from_ISO_string ) n.c = Namespace(doc='c space') n.c.add_option('fred', 'stupid', 'husband from Flintstones') n.c.add_option('wilma', 'waspish', 'wife from Flintstones') n.c.e = Namespace(doc='e space') n.c.e.add_option('dwight', default=97, doc='my uncle') n.c.add_option('dwight', default=98, doc='your uncle') n.d = Namespace(doc='d space') n.d.add_option('fred', 'crabby', 'male neighbor from I Love Lucy') n.d.add_option('ethel', 'silly', 'female neighbor from I Love Lucy') n.x = Namespace(doc='x space') n.x.add_option('size', 100, 'how big in tons', short_form='s') n.x.add_option('password', 'secret', 'the password') return n
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".MutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = { "foo": DotDict({"other": "thing"}), "bar": DotDict({"something": "else"}), } poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") assert processed_crash["foo"]["other"] == "thing" assert processed_crash["bar"]["something"] == "else"
def test_poly_crash_storage_processed_crash_immutability(self): n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".MutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = {"foo": "bar"} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") # It's important to be aware that the only thing # MutatingProcessedCrashCrashStorage class does, in its # save_raw_and_processed() is that it deletes a key called # 'foo'. # This test makes sure that the dict processed_crash here # is NOT affected. assert processed_crash["foo"] == "bar"
def test_basic_usage_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', #default=TransactionExecutorWithBackoff, default=TransactionExecutor, doc='a class that will execute transactions' ) required_config.add_option( 'database_class', default=MockConnectionContext, from_string_converter=class_converter ) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], ) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class(config, mocked_context) _function_calls = [] # some mutable def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) executor(mock_function) self.assertTrue(_function_calls) self.assertEqual(commit_count, 1) self.assertEqual(rollback_count, 0)
def test_poly_crash_storage_processed_crash_immutability(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_namespaces': 'store1', 'store1.crashstorage_class': ('socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage'), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = {'foo': 'bar'} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') # It's important to be aware that the only thing # MutatingProcessedCrashCrashStorage class does, in its # save_raw_and_processed() is that it deletes a key called # 'foo'. # This test makes sure that the dict processed_crash here # is NOT affected. assert processed_crash['foo'] == 'bar'
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_classes': ( 'socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage' ), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = { 'foo': DotDict({'other': 'thing'}), 'bar': SocorroDotDict({'something': 'else'}), } poly_store = config.storage(config) poly_store.save_raw_and_processed( raw_crash, dump, processed_crash, 'n' ) assert processed_crash['foo']['other'] == 'thing' assert processed_crash['bar']['something'] == 'else'
def main(initial_app, values_source_list=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the ' 'application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option("logger", default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{"logger": mock_logging}], argv_source=[], ) with config_manager.context() as config: crashstorage = CrashStorageBase(config, quit_check_callback=fake_quit_check) crashstorage.save_raw_crash({}, "payload", "ooid") crashstorage.save_processed({}) assert_raises(NotImplementedError, crashstorage.get_raw_crash, "ooid") assert_raises(NotImplementedError, crashstorage.get_raw_dump, "ooid") assert_raises(NotImplementedError, crashstorage.get_unredacted_processed, "ooid") assert_raises(NotImplementedError, crashstorage.remove, "ooid") eq_(crashstorage.new_crashes(), []) crashstorage.close()
def test_benchmarking_crashstore(self): required_config = Namespace() mock_logging = Mock() required_config.add_option("logger", default=mock_logging) required_config.update(BenchmarkingCrashStorage.get_required_config()) fake_crash_store = Mock() config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[ {"logger": mock_logging, "wrapped_crashstore": fake_crash_store, "benchmark_tag": "test"} ], argv_source=[], ) with config_manager.context() as config: crashstorage = BenchmarkingCrashStorage(config, quit_check_callback=fake_quit_check) crashstorage.start_timer = lambda: 0 crashstorage.end_timer = lambda: 1 fake_crash_store.assert_called_with(config, fake_quit_check) crashstorage.save_raw_crash({}, "payload", "ooid") crashstorage.wrapped_crashstore.save_raw_crash.assert_called_with({}, "payload", "ooid") mock_logging.debug.assert_called_with("%s save_raw_crash %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.save_processed({}) crashstorage.wrapped_crashstore.save_processed.assert_called_with({}) mock_logging.debug.assert_called_with("%s save_processed %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_crash("uuid") crashstorage.wrapped_crashstore.get_raw_crash.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_crash %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dump("uuid") crashstorage.wrapped_crashstore.get_raw_dump.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dump %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps("uuid") crashstorage.wrapped_crashstore.get_raw_dumps.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dumps %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps_as_files("uuid") crashstorage.wrapped_crashstore.get_raw_dumps_as_files.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dumps_as_files %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_unredacted_processed("uuid") crashstorage.wrapped_crashstore.get_unredacted_processed.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_unredacted_processed %s", "test", 1) mock_logging.debug.reset_mock()
def main(app_object=None): if isinstance(app_object, six.string_types): app_object = class_converter(app_object) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option('application', doc='the fully qualified module or class of the ' 'application', default=app_object, from_string_converter=class_converter ) app_name = getattr(app_object, 'app_name', 'unknown') app_version = getattr(app_object, 'app_version', '0.0') app_description = getattr(app_object, 'app_description', 'no idea') # create an iterable collection of value sources # the order is important as these will supply values for the sources # defined in the_definition_source. The values will be overlain in turn. # First the os.environ values will be applied. Then any values from an ini # file parsed by getopt. Finally any values supplied on the command line # will be applied. value_sources = (ConfigFileFutureProxy, # alias for allowing the user # to specify a config file on # the command line environment, # alias for os.environ command_line) # alias for getopt # set up the manager with the definitions and values # it isn't necessary to provide the app_name because the # app_object passed in or loaded by the ConfigurationManager will alredy # have that information. config_manager = ConfigurationManager(app_definition, value_sources, app_name=app_name, app_version=app_version, app_description=app_description, ) config = config_manager.get_config() app_object = config.admin.application if isinstance(app_object, type): # invocation of the app if the app_object was a class instance = app_object(config) instance.main() elif inspect.ismodule(app_object): # invocation of the app if the app_object was a module app_object.main(config) elif inspect.isfunction(app_object): # invocation of the app if the app_object was a function app_object(config)
def test_operation_error_with_postgres_with_backoff_with_rollback(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithBackoff, #default=TransactionExecutor, doc='a class that will execute transactions' ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{'database_class': MockConnectionContext, 'backoff_delays': [2, 4, 6, 10, 15]}], ) with config_manager.context() as config: executor = config.transaction_executor_class(config) _function_calls = [] # some mutable _sleep_count = [] def mock_function(connection): assert isinstance(connection, MockConnection) connection.transaction_status = \ psycopg2.extensions.TRANSACTION_STATUS_INTRANS _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.OperationalError('Arh!') def mock_sleep(n): _sleep_count.append(n) # monkey patch the sleep function from inside transaction_executor _orig_sleep = socorro.database.transaction_executor.time.sleep socorro.database.transaction_executor.time.sleep = mock_sleep try: executor(mock_function) self.assertTrue(_function_calls) self.assertEqual(commit_count, 1) self.assertEqual(rollback_count, 5) self.assertTrue(mock_logging.warnings) self.assertEqual(len(mock_logging.warnings), 5) self.assertTrue(len(_sleep_count) > 10) finally: socorro.database.transaction_executor.time.sleep = _orig_sleep
def define_config(): definition = Namespace() definition.add_option(name='devowel', default=False, doc='Removes all vowels (including Y)', short_form='d') definition.add_option(name='file', default='', doc='file name for the input text', short_form='f') return definition
def get_standard_config_manager( more_definitions=None, overrides=None, ): # MOCKED CONFIG DONE HERE required_config = Namespace() required_config.add_option( 'logger', default=SilentFakeLogger(), doc='a logger', ) required_config.add_option( 'executor_identity', default=Mock() ) if isinstance(more_definitions, Sequence): definitions = [required_config] definitions.extend(more_definitions) elif more_definitions is not None: definitions = [required_config, more_definitions] else: definitions = [required_config] local_overrides = [ environment, ] if isinstance(overrides, Sequence): overrides.extend(local_overrides) elif overrides is not None: overrides = [overrides].extend(local_overrides) else: overrides = local_overrides config_manager = ConfigurationManager( definitions, values_source_list=overrides, app_name='test-crontabber', app_description=__doc__, argv_source=[] ) # very useful debug #import contextlib #import sys #@contextlib.contextmanager #def stdout_opener(): #yield sys.stdout #config_manager.write_conf('conf', stdout_opener) return config_manager
def define_config(): definition = Namespace() definition.add_option( name='redmine-root', doc='Root url of redmine server', short_form='r' ) definition.add_option( name='redmine-apikey', doc='Redmine API key', short_form='a' ) return definition
def get_logging_namespace(): logging = Namespace() for name, (cls, help_str) in log_formatters.iteritems(): logging.add_option(name='{}'.format(name), doc=help_str, default=None) for optname, (cls, help_str, formatters, action) in \ fmt_options.iteritems(): for fmt in formatters: if fmt in log_formatters: logging.add_option(name='{}-{}'.format(fmt, optname), doc=help_str, default=None) return logging
def test_write_simple(self): rc = Namespace() rc.add_option( 'a', default=23 ) rc.add_option( 'b', default='this is b' ) rc.namespace('n') rc.n.add_option( 'x', default=datetime(1999, 12, 31, 11, 59) ) rc.n.add_option( 'y', default=timedelta(3) ) rc.n.add_option( 'z', default=date(1650, 10, 2) ) cm = ConfigurationManager( [rc], values_source_list=[ { 'a': 68, 'n.x': datetime(1960, 5, 4, 15, 10), 'n.y': timedelta(3), 'n.z': date(2001, 1, 1) } ] ) s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) r = s.getvalue() g = {} l = {} six.exec_(r, g, l) self.assertEqual(l['a'], 68) self.assertEqual(l['b'], 'this is b') self.assertEqual(l['n'].x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(l['n'].y, timedelta(3)) self.assertEqual(l['n'].z, date(2001, 1, 1))
def test_write_simple(self): rc = Namespace() rc.add_option( 'a', default=23 ) rc.add_option( 'b', default='this is b' ) rc.namespace('n') rc.n.add_option( 'x', default=datetime(1999, 12, 31, 11, 59) ) rc.n.add_option( 'y', default=timedelta(3) ) rc.n.add_option( 'z', default=date(1650, 10, 2) ) cm = ConfigurationManager( [rc], values_source_list=[ { 'a': 68, 'n.x': datetime(1960, 5, 4, 15, 10), 'n.y': timedelta(3), 'n.z': date(2001, 1, 1) } ] ) s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) r = s.getvalue() g = {} l = {} exec r in g, l self.assertEqual(l['a'], 68) self.assertEqual(l['b'], 'this is b') self.assertEqual(l['n'].x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(l['n'].y, timedelta(3)) self.assertEqual(l['n'].z, date(2001, 1, 1))
def setup_configman_namespace(self): n = Namespace() n.add_option( 'alpha', default=3, doc='the first parameter', is_argument=True ) n.add_option( 'beta', default='the second', doc='the first parameter', short_form='b', ) n.add_option( 'gamma', default="1 2 3", from_string_converter=quote_stripping_list_of_ints, to_string_converter=partial( list_to_str, delimiter=' ' ), secret=True, ) n.add_option( 'delta', default=False, from_string_converter=boolean_converter ) return n
def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = {"gamma": [38, 28, 18, 8]} other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter("configman.tests.test_val_for_modules.Delta"), "messy": 34, "dd": class_converter("configman.tests.test_val_for_modules.Beta"), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def define_config(): definition = Namespace() definition.add_option( name='devowel', default=False, doc='Removes all vowels (including Y)', short_form='d' ) definition.add_option( name='file', default='', doc='file name for the input text', short_form='f' ) return definition
def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = { "gamma": [38, 28, 18, 8] } other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter ) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter( "configman.tests.test_val_for_modules.Delta" ), "messy": 34, "dd": class_converter( "configman.tests.test_val_for_modules.Beta" ), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def test_classes_in_namespaces_converter_3(self): n = Namespace() n.add_option('kls_list', default='configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha, ' 'configman.tests.test_converters.Alpha', from_string_converter= converters.classes_in_namespaces_converter('kls%d')) cm = ConfigurationManager(n, argv_source=[]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3) for x in config.kls_list.subordinate_namespace_names: self.assertTrue(x in config) self.assertEqual(config[x].cls, Alpha) self.assertTrue('cls_instance' not in config[x])
def test_classes_in_namespaces_converter_3(self): n = Namespace() n.add_option( 'kls_list', default=('socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo, ' 'socorro.unittest.lib.test_converters.Foo'), from_string_converter=str_to_classes_in_namespaces_converter( '%(name)s_%(index)02d')) cm = ConfigurationManager(n, argv_source=[]) config = cm.get_config() self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3) self.assertTrue('Foo_00' in config) self.assertTrue('Foo_01' in config) self.assertTrue('Foo_02' in config)
def test_as_overlay(self): rc = Namespace() rc.add_option( 'a', default=23 ) rc.add_option( 'b', default='this is b' ) rc.namespace('n') rc.n.add_option( 'x', default=datetime(1999, 12, 31, 11, 59) ) rc.n.add_option( 'y', default=timedelta(3) ) rc.n.add_option( 'z', default=date(1650, 10, 2) ) rc.dynamic_load = None cm = ConfigurationManager( [rc], values_source_list=[ 'configman.tests.values_for_module_tests_2', 'configman.tests.values_for_module_tests_3', ] ) config = cm.get_config() self.assertEqual(config.a, 99) self.assertEqual(config.b, 'now is the time') self.assertEqual(config.n.x, datetime(1960, 5, 4, 15, 10)) self.assertEqual(config.n.y, timedelta(1)) self.assertEqual(config.n.z, date(1960, 5, 4)) from configman.tests.values_for_module_tests_3 import Alpha self.assertEqual(config.dynamic_load, Alpha) self.assertEqual(config.host, 'localhost') self.assertEqual(config.port, 5432)
def donttest_write_flat_with_migration(self): n = Namespace() n.add_option('x', default=13, doc='the x') n.add_option('y', default=-1, doc='the y') n.add_option('z', default='fred', doc='the z') n.namespace('o') n.o.add_option('x', default=13, doc='the x') c = ConfigurationManager( [n], use_admin_controls=True, use_auto_help=False, argv_source=[] ) out = StringIO() c.write_conf(for_conf, opener=stringIO_context_wrapper(out)) result = out.getvalue() expected = ( "# name: x\n" "# doc: the x\n" "# converter: int\n" "# x='13'\n" "\n" "# name: y\n" "# doc: the y\n" "# converter: int\n" "y='-1'\n" "\n" "# name: z\n" "# doc: the z\n" "# converter: str\n" "z='fred'\n" "\n" "#-------------------------------------------------------------------------------\n" "# o - \n" "\n" "# name: o.x\n" "# doc: the x\n" "# converter: int\n" "# o.x='13'\n" "\n" ) self.assertEqual(expected, result, "exepected\n%s\nbut got\n%s" % (expected, result))