def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = {"gamma": [38, 28, 18, 8]} other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter("configman.tests.test_val_for_modules.Delta"), "messy": 34, "dd": class_converter("configman.tests.test_val_for_modules.Beta"), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def jobs_converter(path_or_jobs): """Takes a Python dotted path or a jobs spec and returns crontabber jobs Example Python dotted path:: jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS') Example jobs spec:: jobs_converter('socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1h') :arg str path_or_jobs: a Python dotted path or a crontabber jobs spec :returns: crontabber jobs InnerClassList """ if '|' not in path_or_jobs: # NOTE(willkg): configman's class_converter returns the value pointed # to by a Python dotted path input_str = class_converter(path_or_jobs) else: input_str = path_or_jobs from_string_converter = classes_in_namespaces_converter_with_compression( reference_namespace=Namespace(), list_splitter_fn=line_splitter, class_extractor=pipe_splitter, extra_extractor=get_extra_as_options) return from_string_converter(input_str)
def jobs_converter(path_or_jobs): """Takes a Python dotted path or a jobs spec and returns crontabber jobs Example Python dotted path:: jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS') Example jobs spec:: jobs_converter('socorro.cron.jobs.archivescraper.ArchiveScraperCronApp|1h') :arg str path_or_jobs: a Python dotted path or a crontabber jobs spec :returns: crontabber jobs InnerClassList """ if '|' not in path_or_jobs: # NOTE(willkg): configman's class_converter returns the value pointed # to by a Python dotted path input_str = class_converter(path_or_jobs) else: input_str = path_or_jobs # At this point, we have a big string of jobs from_string_converter = classes_in_namespaces_converter_with_compression( reference_namespace=Namespace()) return from_string_converter(input_str)
class HBaseClientApp(generic_app.App): app_name = "hbase_client.py" app_version = "0.1" app_description = __doc__ required_config = Namespace() required_config.add_option( 'hbase_crash_storage_class', default=HappyBaseCrashStorage, doc='the class responsible for proving an hbase connection', from_string_converter=class_converter) required_config.add_option( 'command', default=help, doc='command to use', is_argument=True, from_string_converter=lambda s: class_converter(__name__ + '.' + s)) required_config.add_option( 'json', default=False, short_form='j', doc='json output instead of a pretty printed mapping', ) def main(self): self.storage = self.config.hbase_crash_storage_class(self.config) self.config.command(self).run()
def main(initial_app, values_source_list=None, config_path=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) if config_path is None: default = './config' config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', default) if config_path != default: # you tried to set it, then it must be a valid directory if not os.path.isdir(config_path): raise IOError('%s is not a valid directory' % config_path) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def main(initial_app, values_source_list=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the ' 'application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
class InnerClassList(RequiredConfig): """This nested class is a proxy list for the classes. It collects all the config requirements for the listed classes and places them each into their own Namespace. """ # we're dynamically creating a class here. The following block of # code is actually adding class level attributes to this new class # 1st requirement for configman required_config = Namespace() # to help the programmer know what Namespaces we added subordinate_namespace_names = [] # save the template for future reference namespace_template = template_for_namespace # for display original_input = class_list_str.replace('\n', '\\n') # for each class in the class list class_list = [] for namespace_index, class_list_element in enumerate( class_str_list): try: a_class = class_converter( class_extractor(class_list_element)) except AttributeError: raise JobNotFoundError(class_list_element) class_list.append((a_class.__name__, a_class)) # figure out the Namespace name namespace_name_dict = { 'name': a_class.__name__, 'index': namespace_index } namespace_name = template_for_namespace % namespace_name_dict subordinate_namespace_names.append(namespace_name) # create the new Namespace required_config.namespace(namespace_name) a_class_namespace = required_config[namespace_name] # add options for the 'extra data' try: extra_options = extra_extractor(class_list_element) a_class_namespace.update(extra_options) except NotImplementedError: pass # add options frr the classes required config try: for k, v in a_class.get_required_config().iteritems(): if k not in reference_namespace: a_class_namespace[k] = v except AttributeError: # a_class has no get_required_config pass @classmethod def to_str(cls): """this method takes this inner class object and turns it back into the original string of classnames. This is used primarily as for the output of the 'help' option""" return cls.original_input
def jobs_converter(path_or_jobs): """Takes a Python dotted path or a jobs spec and returns crontabber jobs Example Python dotted path:: jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS') Example jobs spec:: jobs_converter('socorro.cron.jobs.archivescraper.ArchiveScraperCronApp|1h') :arg str path_or_jobs: a Python dotted path or a crontabber jobs spec :returns: crontabber jobs InnerClassList """ if '|' not in path_or_jobs: # NOTE(willkg): configman's class_converter returns the value pointed # to by a Python dotted path input_str = class_converter(path_or_jobs) else: input_str = path_or_jobs # At this point, we have a big string of jobs from_string_converter = classes_in_namespaces_converter_with_compression( reference_namespace=Namespace() ) return from_string_converter(input_str)
def main(initial_app, values_source_list=None, config_path=None): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) if config_path is None: default = './config' config_path = os.environ.get( 'DEFAULT_SOCORRO_CONFIG_PATH', default ) if config_path != default: # you tried to set it, then it must be a valid directory if not os.path.isdir(config_path): raise IOError('%s is not a valid directory' % config_path) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def test_basic_07_argparse_multilevel_class_expansion(self): option_definitions = self.setup_configman_namespace() other_value_source = { "gamma": [38, 28, 18, 8] } other_definition_source = Namespace() other_definition_source.add_option( "a_class", default="configman.tests.test_val_for_modules.Alpha", from_string_converter=class_converter ) cm = ConfigurationManager( definition_source=[option_definitions, other_definition_source], values_source_list=[other_value_source, command_line], argv_source=[ "0", "--admin.expose_secrets", "--delta", '--gamma="8 18 28 38"', '--a_class=configman.tests.test_val_for_modules.Delta', '--messy=34' ], use_auto_help=False, ) config = cm.get_config() expected = { "alpha": 0, "beta": 'the second', "gamma": [8, 18, 28, 38], "delta": True, "admin.print_conf": None, "admin.dump_conf": '', "admin.strict": False, "admin.expose_secrets": True, "a_class": class_converter( "configman.tests.test_val_for_modules.Delta" ), "messy": 34, "dd": class_converter( "configman.tests.test_val_for_modules.Beta" ), 'b': 23, } for k in config.keys_breadth_first(): self.assertEqual(config[k], expected[k])
def main(app_object=None): if isinstance(app_object, basestring): app_object = class_converter(app_object) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.admin = admin = Namespace() admin.add_option('application', doc='the fully qualified module or class of the ' 'application', default=app_object, from_string_converter=class_converter ) app_name = getattr(app_object, 'app_name', 'unknown') app_version = getattr(app_object, 'app_version', '0.0') app_description = getattr(app_object, 'app_description', 'no idea') # create an iterable collection of value sources # the order is important as these will supply values for the sources # defined in the_definition_source. The values will be overlain in turn. # First the os.environ values will be applied. Then any values from an ini # file parsed by getopt. Finally any values supplied on the command line # will be applied. value_sources = (ConfigFileFutureProxy, # alias for allowing the user # to specify a config file on # the command line environment, # alias for os.environ command_line) # alias for getopt # set up the manager with the definitions and values # it isn't necessary to provide the app_name because the # app_object passed in or loaded by the ConfigurationManager will alredy # have that information. config_manager = ConfigurationManager(app_definition, value_sources, app_name=app_name, app_version=app_version, app_description=app_description, ) config = config_manager.get_config() app_object = config.admin.application if isinstance(app_object, type): # invocation of the app if the app_object was a class instance = app_object(config) instance.main() elif inspect.ismodule(app_object): # invocation of the app if the app_object was a module app_object.main(config) elif inspect.isfunction(app_object): # invocation of the app if the app_object was a function app_object(config)
def main(app_object=None): if isinstance(app_object, basestring): app_object = class_converter(app_object) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.admin = admin = Namespace() admin.add_option('application', doc='the fully qualified module or class of the ' 'application', default=app_object, from_string_converter=class_converter) app_name = getattr(app_object, 'app_name', 'unknown') app_version = getattr(app_object, 'app_version', '0.0') app_description = getattr(app_object, 'app_description', 'no idea') # create an iterable collection of value sources # the order is important as these will supply values for the sources # defined in the_definition_source. The values will be overlain in turn. # First the os.environ values will be applied. Then any values from an ini # file parsed by getopt. Finally any values supplied on the command line # will be applied. value_sources = ( ConfigFileFutureProxy, # alias for allowing the user # to specify a config file on # the command line environment, # alias for os.environ command_line) # alias for getopt # set up the manager with the definitions and values # it isn't necessary to provide the app_name because the # app_object passed in or loaded by the ConfigurationManager will alredy # have that information. config_manager = ConfigurationManager( app_definition, value_sources, app_name=app_name, app_version=app_version, app_description=app_description, ) config = config_manager.get_config() app_object = config.admin.application if isinstance(app_object, type): # invocation of the app if the app_object was a class instance = app_object(config) instance.main() elif inspect.ismodule(app_object): # invocation of the app if the app_object was a module app_object.main(config) elif inspect.isfunction(app_object): # invocation of the app if the app_object was a function app_object(config)
def action_converter(action): try: return action_dispatch[action] except KeyError: try: f = class_converter(action) except Exception: raise Exception("'%s' is not a valid action" % action) if f in action_dispatch.values(): return f raise Exception("'%s' is not a valid action" % action)
def __init__(self, source, the_config_manager=None): if isinstance(source, (six.binary_type, six.text_type)): source = class_converter(source) module_as_dotdict = DotDict() try: ignore_symbol_list = source.ignore_symbol_list if 'ignore_symbol_list' not in ignore_symbol_list: ignore_symbol_list.append('ignore_symbol_list') except AttributeError: ignore_symbol_list = [] try: self.always_ignore_mismatches = source.always_ignore_mismatches except AttributeError: pass # don't need to do anything - mismatches will not be ignored for key, value in six.iteritems(source.__dict__): if key.startswith('__') and key != "__doc__": continue if key in ignore_symbol_list: continue module_as_dotdict[key] = value self.module = source self.source = module_as_dotdict
def main(initial_app, values_source_list=None, config_path='./config'): if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError, x: raise AppDetailMissingError(str(x))
def _do_main( initial_app, values_source_list=None, config_path=None, config_manager_cls=ConfigurationManager ): global restart restart = False if isinstance(initial_app, basestring): initial_app = class_converter(initial_app) if config_path is None: default = './config' config_path = os.environ.get( 'DEFAULT_SOCORRO_CONFIG_PATH', default ) if config_path != default: # you tried to set it, then it must be a valid directory if not os.path.isdir(config_path): raise IOError('%s is not a valid directory' % config_path) # the only config parameter is a special one that refers to a class or # module that defines an application. In order to qualify, a class must # have a constructor that accepts a DotDict derivative as the sole # input parameter. It must also have a 'main' function that accepts no # parameters. For a module to be acceptable, it must have a main # function that accepts a DotDict derivative as its input parameter. app_definition = Namespace() app_definition.add_option( 'application', doc='the fully qualified module or class of the application', default=initial_app, from_string_converter=class_converter ) try: app_name = initial_app.app_name # this will be used as the default # b app_version = initial_app.app_version app_description = initial_app.app_description except AttributeError as x: raise AppDetailMissingError(x) app_definition.add_aggregation( 'logger', functools.partial(setup_logger, app_name) ) definitions = ( app_definition, logging_required_config(app_name) ) config_manager = config_manager_cls( definitions, app_name=app_name, app_version=app_version, app_description=app_description, values_source_list=values_source_list, config_pathname=config_path ) def fix_exit_code(code): # some apps don't return a code so you might get None # which isn't good enough to send to sys.exit() if code is None: return 0 return code with config_manager.context() as config: #config.logger.config = config config.executor_identity = lambda: threading.currentThread().getName() config_manager.log_config(config.logger) # install the signal handler for SIGHUP to be the action defined in # 'respond_to_SIGHUP' respond_to_SIGHUP_with_logging = functools.partial( respond_to_SIGHUP, logger=config.logger ) signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging) # get the app class from configman. Why bother since we have it aleady # with the 'initial_app' name? In most cases initial_app == app, # it might not always be that way. The user always has the ability # to specify on the command line a new app class that will override # 'initial_app'. app = config.application if isinstance(app, type): # invocation of the app if the app_object was a class instance = app(config) instance.config_manager = config_manager return_code = fix_exit_code(instance.main()) elif inspect.ismodule(app): # invocation of the app if the app_object was a module return_code = fix_exit_code(app.main(config)) elif inspect.isfunction(app): # invocation of the app if the app_object was a function return_code = fix_exit_code(app(config)) config.logger.info('done.') return return_code raise NotImplementedError("The app did not have a callable main function")