class ConfigurationManager(object): #-------------------------------------------------------------------------- def __init__( self, definition_source=None, values_source_list=None, argv_source=None, #use_config_files=True, use_auto_help=True, use_admin_controls=True, quit_after_admin=True, options_banned_from_help=None, app_name='', app_version='', app_description='', config_pathname='.', config_optional=True, ): """create and initialize a configman object. parameters: definition_source - a namespace or list of namespaces from which configman is to fetch the definitions of the configuration parameters. values_source_list - (optional) a hierarchical list of sources for values for the configuration parameters. As values are copied from these sources, conficting values are resolved with sources on the right getting preference over sources on the left. argv_source - if the values_source_list contains a commandline source, this value is an alternative source for actual command line arguments. Useful for testing or preprocessing command line arguments. use_auto_help - set to True if configman is to automatically set up help output for command line invocations. use_admin_controls - configman can add command line flags that it interprets independently of the app defined arguments. True enables this capability, while, False supresses it. quit_after_admin - if True and admin controls are enabled and used, call sys.exit to end the app. This is useful to stop the app from running if all that was done was to write a config file or stop after help. options_banned_from_help - a list of strings that will censor the output of help to prevent specified options from being listed in the help output. This is useful for hiding debug or secret command line arguments. app_name - assigns a name to the app. This is used in help output and as a default basename for config files. app_version - assigns a version for the app used help output. app_description - assigns a description for the app to be used in the help output. config_pathname - a hard coded path to the directory of or the full path and name of the configuration file. config_optional - a boolean indicating if a missing default config file is optional. Note: this is only for the default config file. If a config file is specified on the commandline, it _must_ exsist.""" # instead of allowing mutables as default keyword argument values... if definition_source is None: definition_source_list = [] elif (isinstance(definition_source, collections.Sequence) and not isinstance(definition_source, basestring)): definition_source_list = list(definition_source) else: definition_source_list = [definition_source] if argv_source is None: self.argv_source = sys.argv[1:] self.app_invocation_name = sys.argv[0] else: self.argv_source = argv_source self.app_invocation_name = app_name if options_banned_from_help is None: options_banned_from_help = ['application'] self.config_pathname = config_pathname self.config_optional = config_optional self.app_name = app_name self.app_version = app_version self.app_description = app_description self.args = [] # extra commandline arguments that are not switches # will be stored here. self._config = None # eventual container for DOM-like config object self.option_definitions = Namespace() self.definition_source_list = definition_source_list if values_source_list is None: # nothing set, assume defaults if use_admin_controls: values_source_list = (cm.ConfigFileFutureProxy, cm.environment, cm.command_line) else: values_source_list = (cm.environment, cm.command_line) admin_tasks_done = False self.admin_controls_list = [ 'help', 'admin.conf', 'admin.dump_conf', 'admin.print_conf', 'admin.migration', 'admin.strict' ] self.options_banned_from_help = options_banned_from_help if use_auto_help: self._setup_auto_help() if use_admin_controls: admin_options = self._setup_admin_options(values_source_list) self.definition_source_list.append(admin_options) # iterate through the option definitions to create the nested dict # hierarchy of all the options called 'option_definitions' for a_definition_source in self.definition_source_list: try: safe_copy_of_def_source = a_definition_source.safe_copy() except AttributeError: # apparently, the definition source was not in the form of a # Namespace object. This isn't a show stopper, but we don't # know how to make a copy of this object safely: we know from # experience that the stock copy.copy method leads to grief # as many sub-objects within an option definition source can # not be copied that way (classes, for example). # The only action we can take is to trust and continue with the # original copy of the definition source. safe_copy_of_def_source = a_definition_source def_sources.setup_definitions(safe_copy_of_def_source, self.option_definitions) if use_admin_controls: # the name of the config file needs to be loaded from the command # line prior to processing the rest of the command line options. config_filename = \ value_sources.config_filename_from_commandline(self) if (config_filename and cm.ConfigFileFutureProxy in values_source_list): self.option_definitions.admin.conf.default = config_filename self.values_source_list = value_sources.wrap(values_source_list, self) known_keys = self._overlay_expand() self._check_for_mismatches(known_keys) # the app_name, app_version and app_description are to come from # if 'application' option if it is present. If it is not present, # get the app_name,et al, from parameters passed into the constructor. # if those are empty, set app_name, et al, to empty strings try: app_option = self._get_option('application') self.app_name = getattr(app_option.value, 'app_name', '') self.app_version = getattr(app_option.value, 'app_version', '') self.app_description = getattr(app_option.value, 'app_description', '') except exc.NotAnOptionError: # there is no 'application' option, continue to use the # 'app_name' from the parameters passed in, if they exist. pass if use_auto_help and self._get_option('help').value: self.output_summary() admin_tasks_done = True if use_admin_controls and self._get_option('admin.print_conf').value: self.print_conf() admin_tasks_done = True if use_admin_controls and self._get_option('admin.dump_conf').value: self.dump_conf() admin_tasks_done = True if quit_after_admin and admin_tasks_done: sys.exit() #-------------------------------------------------------------------------- @contextlib.contextmanager def context(self): """return a config as a context that calls close on every item when it goes out of scope""" config = None try: config = self.get_config() yield config finally: if config: self._walk_and_close(config) #-------------------------------------------------------------------------- def get_config(self, mapping_class=DotDictWithAcquisition): config = self._generate_config(mapping_class) if self._aggregate(self.option_definitions, config, config): # state changed, must regenerate return self._generate_config(mapping_class) else: return config #-------------------------------------------------------------------------- def output_summary(self, output_stream=sys.stdout, block_password=True): """outputs a usage tip and the list of acceptable commands. This is useful as the output of the 'help' option. parameters: output_stream - an open file-like object suitable for use as the target of a print statement block_password - a boolean driving the use of a string of * in place of the value for any object containing the substring 'passowrd' """ if self.app_name or self.app_description: print >> output_stream, 'Application:', if self.app_name: print >> output_stream, self.app_name, self.app_version if self.app_description: print >> output_stream, self.app_description if self.app_name or self.app_description: print >> output_stream, '' names_list = self.get_option_names() print >> output_stream, "usage:\n", self.app_invocation_name, \ "[OPTIONS]...", bracket_count = 0 for key in names_list: an_option = self.option_definitions[key] if an_option.is_argument: if an_option.default is None: print >> output_stream, an_option.name, else: print >> output_stream, "[ %s" % an_option.name, bracket_count += 1 print >> output_stream, ']' * bracket_count, '\n' names_list.sort() if names_list: print >> output_stream, 'OPTIONS:' pad = ' ' * 4 for name in names_list: if name in self.options_banned_from_help: continue option = self._get_option(name) line = ' ' * 2 # always start with 2 spaces if option.short_form: line += '-%s, ' % option.short_form line += '--%s' % name line += '\n' doc = option.doc if option.doc is not None else '' if doc: line += '%s%s\n' % (pad, doc) try: value = option.value type_of_value = type(value) converter_function = conv.to_string_converters[type_of_value] default = converter_function(value) except KeyError: default = option.value if default is not None: if 'password' in name.lower(): default = '*********' if name not in ('help', ): # don't bother with certain dead obvious ones line += '%s(default: %s)\n' % (pad, default) print >> output_stream, line #-------------------------------------------------------------------------- def print_conf(self): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" config_file_type = self._get_option('admin.print_conf').value @contextlib.contextmanager def stdout_opener(): yield sys.stdout skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_print_conf ] self.write_conf(config_file_type, stdout_opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def dump_conf(self, config_pathname=None): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" if not config_pathname: config_pathname = self._get_option('admin.dump_conf').value opener = functools.partial(open, config_pathname, 'w') config_file_type = os.path.splitext(config_pathname)[1][1:] skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_dump_conf ] self.write_conf(config_file_type, opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def write_conf(self, config_file_type, opener, skip_keys=None): """write a configuration file to a file-like object. parameters: config_file_type - a string containing a registered file type OR a for_XXX module from the value_source package. Passing in an string that is unregistered will result in a KeyError opener - a callable object or function that returns a file like object that works as a context in a with statement.""" blocked_keys = self.admin_controls_list if skip_keys: blocked_keys.extend(skip_keys) if blocked_keys: option_defs = self.option_definitions.safe_copy() for a_blocked_key in blocked_keys: try: del option_defs[a_blocked_key] except (AttributeError, KeyError): # okay that key isn't here pass # remove empty namespaces all_keys = [ k for k in option_defs.keys_breadth_first(include_dicts=True) ] for key in all_keys: candidate = option_defs[key] if (isinstance(candidate, Namespace) and not len(candidate)): del option_defs[key] else: option_defs = self.option_definitions if self.option_definitions.admin.migration.default: self._migrate_options_for_acquisition(option_defs) value_sources.write(config_file_type, option_defs, opener) #-------------------------------------------------------------------------- @staticmethod def _migrate_options_for_acquisition(option_defs): """sift through the definitions looking for common keys that can be migrated to a lower level in the hierarchy. create a mapping with these characteristics: key - composed of the values from Option objects for 'name', 'default' and 'from_string' converstion function. value - a list of the the keys from the currently active option definition. this implements a reverse index of value to keys for the system of Namespaces and Options that make up the configuration manager's option definition mapping. the mapping is keyed by the option name, its default value and the from_string_converter function. If all these things are the same in two options, then it is considered that they are referring to the same option. This means that that option may be able migrate to a lower level. The mitigating factor is the number of Options with the same key name. If there is more than one with the same name, then the Option cannot migrate to a lower level. """ migration_candidates = collections.defaultdict(list) option_name_counts = collections.defaultdict(int) # cycle through all the keys for option_name in option_defs.keys_breadth_first(): # option_name is a fully qualified key: x.y.z an_option = option_defs[option_name] if isinstance(an_option, Option): name_default_converter_key = ( an_option.name, str(an_option.default), str(an_option.from_string_converter)) if name_default_converter_key not in migration_candidates: option_name_counts[an_option.name] += 1 migration_candidates[name_default_converter_key].append( option_name) for candidate, original_keys in migration_candidates.iteritems(): # candidate is: (option_name, option_default, option_coverter) # remove name qualifications: x.y.z --> z if (option_name_counts[candidate[0]] == 1 and len(original_keys) > 1): option_defs[candidate[0]] = \ option_defs[original_keys[0]].copy() option_defs[candidate[0]].not_for_definition = True for a_key in original_keys: option_defs[a_key].comment_out = True #-------------------------------------------------------------------------- def log_config(self, logger): """write out the current configuration to a log-like object. parameters: logger - a object that implements a method called 'info' with the same semantics as the call to 'logger.info'""" logger.info("app_name: %s", self.app_name) logger.info("app_version: %s", self.app_version) logger.info("current configuration:") config = [(key, self.option_definitions[key].value) for key in self.option_definitions.keys_breadth_first() if key not in self.admin_controls_list] config.sort() for key, val in config: if 'password' in key.lower(): logger.info('%s: *********', key) else: try: logger.info('%s: %s', key, conv.to_string_converters[type(key)](val)) except KeyError: logger.info('%s: %s', key, val) #-------------------------------------------------------------------------- def get_option_names(self): """returns a list of fully qualified option names. returns: a list of strings representing the Options in the source Namespace list. Each item will be fully qualified with dot delimited Namespace names. """ return [ x for x in self.option_definitions.keys_breadth_first() if isinstance(self.option_definitions[x], Option) ] #-------------------------------------------------------------------------- def _overlay_expand(self): """This method overlays each of the value sources onto the default in each of the defined options. It does so using a breadth first iteration, overlaying and expanding each level of the tree in turn. As soon as no changes were made to any level, the loop breaks and the work is done. The actual action of the overlay is to take the value from the source and copy into the 'default' member of each Option object. "expansion" means converting an option value into its real type from string. The conversion is accomplished by simply calling the 'set_value' method of the Option object. If the resultant type has its own configuration options, bring those into the current namespace and then proceed to overlay/expand those. """ new_keys_discovered = True # loop control, False breaks the loop known_keys = set() # a set of keys that have been expanded while new_keys_discovered: # loop until nothing more is done # keys holds a list of all keys in the option definitons in # breadth first order using this form: [ 'x', 'y', 'z', 'x.a', # 'x.b', 'z.a', 'z.b', 'x.a.j', 'x.a.k', 'x.b.h'] keys = [x for x in self.option_definitions.keys_breadth_first()] new_keys_discovered = False # setup to break loop # overlay process: # fetch all the default values from the value sources before # applying the from string conversions for key in keys: if key not in known_keys: # skip all keys previously seen # loop through all the value sources looking for values # that match this current key. for a_value_source in self.values_source_list: try: # get all the option values from this value source val_src_dict = a_value_source.get_values( self, True) # make sure it is in the form of a DotDict if not isinstance(val_src_dict, DotDict): val_src_dict = \ DotDictWithAcquisition(val_src_dict) # get the Option for this key opt = self.option_definitions[key] # overlay the default with the new value from # the value source. This assignment may come # via acquisition, so the key given may not have # been an exact match for what was returned. opt.default = val_src_dict[key] except KeyError, x: pass # okay, that source doesn't have this value # expansion process: # step through all the keys converting them to their proper # types and bringing in any new keys in the process for key in keys: if key not in known_keys: # skip all keys previously seen # mark this key as having been seen and processed known_keys.add(key) an_option = self.option_definitions[key] if isinstance(an_option, Aggregation): continue # aggregations are ignored # apply the from string conversion to make the real value an_option.set_value(an_option.default) # new values have been seen, don't let loop break new_keys_discovered = True try: try: # try to fetch new requirements from this value new_req = an_option.value.get_required_config() except AttributeError: new_req = an_option.value.required_config # make sure what we got as new_req is actually a # Mapping of some sort if not isinstance(new_req, collections.Mapping): # we didn't get a mapping, perhaps the option value # was a Mock object - in any case we can't try to # interpret 'new_req' as a configman requirement # collection. We must abandon processing this # option further continue # get the parent namespace current_namespace = self.option_definitions.parent(key) if current_namespace is None: # we're at the top level, use the base namespace current_namespace = self.option_definitions # add the new Options to the namespace current_namespace.update(new_req.safe_copy()) except AttributeError, x: # there are apparently no new Options to bring in from # this option's value pass
class ConfigurationManager(object): # -------------------------------------------------------------------------- def __init__( self, definition_source=None, values_source_list=None, argv_source=None, # use_config_files=True, use_auto_help=True, use_admin_controls=True, quit_after_admin=True, options_banned_from_help=None, app_name="", app_version="", app_description="", config_pathname=".", ): """create and initialize a configman object. parameters: definition_source - a namespace or list of namespaces from which configman is to fetch the definitions of the configuration parameters. values_source_list - (optional) a hierarchical list of sources for values for the configuration parameters. As values are copied from these sources, conficting values are resolved with sources on the right getting preference over sources on the left. argv_source - if the values_source_list contains a commandline source, this value is an alternative source for actual command line arguments. Useful for testing or preprocessing command line arguments. use_auto_help - set to True if configman is to automatically set up help output for command line invocations. use_admin_controls - configman can add command line flags that it interprets independently of the app defined arguments. True enables this capability, while, False supresses it. quit_after_admin - if True and admin controls are enabled and used, call sys.exit to end the app. This is useful to stop the app from running if all that was done was to write a config file or stop after help. options_banned_from_help - a list of strings that will censor the output of help to prevent specified options from being listed in the help output. This is useful for hiding debug or secret command line arguments. app_name - assigns a name to the app. This is used in help output and as a default basename for config files. app_version - assigns a version for the app used help output. app_description - assigns a description for the app to be used in the help output. config_pathname - a hard coded path to the directory of or the full path and name of the configuration file.""" # instead of allowing mutables as default keyword argument values... if definition_source is None: definition_source_list = [] elif isinstance(definition_source, collections.Sequence) and not isinstance(definition_source, basestring): definition_source_list = list(definition_source) else: definition_source_list = [definition_source] if argv_source is None: argv_source = sys.argv[1:] if options_banned_from_help is None: options_banned_from_help = ["admin.application"] self.config_pathname = config_pathname self.app_name = app_name self.app_version = app_version self.app_description = app_description self.args = [] # extra commandline arguments that are not switches # will be stored here. self._config = None # eventual container for DOM-like config object self.argv_source = argv_source self.option_definitions = Namespace() self.definition_source_list = definition_source_list if values_source_list is None: # nothing set, assume defaults if use_admin_controls: values_source_list = (cm.ConfigFileFutureProxy, cm.environment, cm.command_line) else: values_source_list = (cm.environment, cm.command_line) admin_tasks_done = False self.admin_controls_list = ["help", "admin.conf", "admin.dump_conf", "admin.print_conf", "admin.application"] self.options_banned_from_help = options_banned_from_help if use_auto_help: self._setup_auto_help() if use_admin_controls: admin_options = self._setup_admin_options(values_source_list) self.definition_source_list.append(admin_options) # iterate through the option definitions to create the nested dict # hierarchy of all the options called 'option_definitions' for a_definition_source in self.definition_source_list: def_sources.setup_definitions(a_definition_source, self.option_definitions) if use_admin_controls: # some admin options need to be loaded from the command line # prior to processing the rest of the command line options. admin_options = value_sources.get_admin_options_from_command_line(self) # integrate the admin_options with 'option_definitions' self._overlay_value_sources_recurse(source=admin_options, ignore_mismatches=True) self.values_source_list = value_sources.wrap(values_source_list, self) # first pass to get classes & config path - ignore bad options self._overlay_value_sources(ignore_mismatches=True) # walk tree expanding class options self._walk_expanding_class_options() # the app_name, app_version and app_description are to come from # if 'admin.application' option if it is present. If it is not present, # get the app_name,et al, from parameters passed into the constructor. # if those are empty, set app_name, et al, to empty strings try: app_option = self._get_option("admin.application") self.app_name = getattr(app_option.value, "app_name", "") self.app_version = getattr(app_option.value, "app_version", "") self.app_description = getattr(app_option.value, "app_description", "") except exc.NotAnOptionError: # there is no 'admin.application' option, continue to use the # 'app_name' from the parameters passed in, if they exist. pass # second pass to include config file values - ignore bad options self._overlay_value_sources(ignore_mismatches=True) # walk tree expanding class options self._walk_expanding_class_options() # third pass to get values - complain about bad options self._overlay_value_sources(ignore_mismatches=False) if use_auto_help and self._get_option("help").value: self.output_summary() admin_tasks_done = True if use_admin_controls and self._get_option("admin.print_conf").value: self.print_conf() admin_tasks_done = True if use_admin_controls and self._get_option("admin.dump_conf").value: self.dump_conf() admin_tasks_done = True if quit_after_admin and admin_tasks_done: sys.exit() # -------------------------------------------------------------------------- @contextlib.contextmanager def context(self): """return a config as a context that calls close on every item when it goes out of scope""" config = None try: config = self.get_config() yield config finally: if config: self._walk_and_close(config) # -------------------------------------------------------------------------- def get_config(self, mapping_class=DotDictWithAcquisition): config = self._generate_config(mapping_class) if self._aggregate(self.option_definitions, config, config): # state changed, must regenerate return self._generate_config(mapping_class) else: return config # -------------------------------------------------------------------------- def output_summary(self, output_stream=sys.stdout, block_password=True): """outputs a usage tip and the list of acceptable commands. This is useful as the output of the 'help' option. parameters: output_stream - an open file-like object suitable for use as the target of a print statement block_password - a boolean driving the use of a string of * in place of the value for any object containing the substring 'passowrd' """ if self.app_name or self.app_description: print >> output_stream, "Application:", if self.app_name: print >> output_stream, self.app_name, self.app_version if self.app_description: print >> output_stream, self.app_description if self.app_name or self.app_description: print >> output_stream, "" names_list = self.get_option_names() names_list.sort() if names_list: print >> output_stream, "Options:" for name in names_list: if name in self.options_banned_from_help: continue option = self._get_option(name) line = " " * 2 # always start with 2 spaces if option.short_form: line += "-%s, " % option.short_form line += "--%s" % name line = line.ljust(30) # seems to the common practise doc = option.doc if option.doc is not None else "" try: value = option.value type_of_value = type(value) converter_function = conv.to_string_converters[type_of_value] default = converter_function(value) except KeyError: default = option.value if default is not None: if "password" in name.lower(): default = "*********" if doc: doc += " " if name not in ("help",): # don't bother with certain dead obvious ones doc += "(default: %s)" % default line += doc print >> output_stream, line # -------------------------------------------------------------------------- def print_conf(self): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" config_file_type = self._get_option("admin.print_conf").value @contextlib.contextmanager def stdout_opener(): yield sys.stdout skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_print_conf ] self.write_conf(config_file_type, stdout_opener, skip_keys=skip_keys) # -------------------------------------------------------------------------- def dump_conf(self, config_pathname=None): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" if not config_pathname: config_pathname = self._get_option("admin.dump_conf").value opener = functools.partial(open, config_pathname, "w") config_file_type = os.path.splitext(config_pathname)[1][1:] skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_dump_conf ] self.write_conf(config_file_type, opener, skip_keys=skip_keys) # -------------------------------------------------------------------------- def write_conf(self, config_file_type, opener, skip_keys=None): """write a configuration file to a file-like object. parameters: config_file_type - a string containing a registered file type OR a for_XXX module from the value_source package. Passing in an string that is unregistered will result in a KeyError opener - a callable object or function that returns a file like object that works as a context in a with statement.""" blocked_keys = self.admin_controls_list if skip_keys: blocked_keys.extend(skip_keys) option_iterator = functools.partial(self._walk_config, blocked_keys=blocked_keys) with opener() as config_fp: value_sources.write(config_file_type, option_iterator, config_fp) # -------------------------------------------------------------------------- def log_config(self, logger): """write out the current configuration to a log-like object. parameters: logger - a object that implements a method called 'info' with the same semantics as the call to 'logger.info'""" logger.info("app_name: %s", self.app_name) logger.info("app_version: %s", self.app_version) logger.info("current configuration:") config = [ (qkey, val.value) for qkey, key, val in self._walk_config(self.option_definitions) if qkey not in self.admin_controls_list and not isinstance(val, Namespace) ] config.sort() for key, val in config: if "password" in key.lower(): logger.info("%s: *********", key) else: try: logger.info("%s: %s", key, conv.to_string_converters[type(key)](val)) except KeyError: logger.info("%s: %s", key, val) # -------------------------------------------------------------------------- def get_option_names(self, source=None, names=None, prefix=""): """returns a list of fully qualified option names. parameters: source - a sequence of Namespace of Options, usually not specified, If not specified, the function will default to using the internal list of Option definitions. names - a list to start with for appending the lsit Option names. If ommited, the function will start with an empty list. returns: a list of strings representing the Options in the source Namespace list. Each item will be fully qualified with dot delimited Namespace names. """ if not source: source = self.option_definitions if names is None: names = [] for key, val in source.items(): if isinstance(val, Namespace): new_prefix = "%s%s." % (prefix, key) self.get_option_names(val, names, new_prefix) elif isinstance(val, Option): names.append("%s%s" % (prefix, key)) # skip aggregations, we want only Options return names # -------------------------------------------------------------------------- @staticmethod def _walk_and_close(a_dict): for val in a_dict.itervalues(): if isinstance(val, collections.Mapping): ConfigurationManager._walk_and_close(val) if hasattr(val, "close") and not inspect.isclass(val): val.close() # -------------------------------------------------------------------------- def _generate_config(self, mapping_class): """This routine generates a copy of the DotDict based config""" config = mapping_class() self._walk_config_copy_values(self.option_definitions, config, mapping_class) return config # -------------------------------------------------------------------------- def _walk_expanding_class_options(self, source_namespace=None, parent_namespace=None): if source_namespace is None: source_namespace = self.option_definitions expanded_keys = [] expansions_were_done = True while expansions_were_done: expansions_were_done = False # can't use iteritems in loop, we're changing the dict for key, val in source_namespace.items(): if isinstance(val, Namespace): self._walk_expanding_class_options(source_namespace=val, parent_namespace=source_namespace) elif key not in expanded_keys and (inspect.isclass(val.value) or inspect.ismodule(val.value)): expanded_keys.append(key) expansions_were_done = True if key == "application": target_namespace = parent_namespace else: target_namespace = source_namespace try: for o_key, o_val in val.value.get_required_config().iteritems(): target_namespace.__setattr__(o_key, copy.deepcopy(o_val)) except AttributeError: pass # there are no required_options for this class else: pass # don't need to touch other types of Options self._overlay_value_sources(ignore_mismatches=True) # -------------------------------------------------------------------------- def _setup_auto_help(self): help_option = Option(name="help", doc="print this", default=False) self.definition_source_list.append({"help": help_option}) # -------------------------------------------------------------------------- def _get_config_pathname(self): if os.path.isdir(self.config_pathname): # we've got a path with no file name at the end # use the appname as the file name and default to an 'ini' # config file type if self.app_name: return os.path.join(self.config_pathname, "%s.ini" % self.app_name) else: # there is no app_name yet # we'll punt and use 'config' return os.path.join(self.config_pathname, "config.ini") return self.config_pathname # -------------------------------------------------------------------------- def _setup_admin_options(self, values_source_list): base_namespace = Namespace() base_namespace.admin = admin = Namespace() admin.add_option( name="print_conf", default=None, doc="write current config to stdout (%s)" % ", ".join(value_sources.file_extension_dispatch.keys()), ) admin.add_option(name="dump_conf", default="", doc="a pathname to which to write the current config") # only offer the config file admin options if they've been requested in # the values source list if ConfigFileFutureProxy in values_source_list: default_config_pathname = self._get_config_pathname() admin.add_option( name="conf", default=default_config_pathname, doc="the pathname of the config file " "(path/filename)" ) return base_namespace # -------------------------------------------------------------------------- def _overlay_value_sources(self, ignore_mismatches=True): for a_settings_source in self.values_source_list: try: this_source_ignore_mismatches = ignore_mismatches or a_settings_source.always_ignore_mismatches except AttributeError: # the settings source doesn't have the concept of always # ignoring mismatches, so the original value of # ignore_mismatches stands this_source_ignore_mismatches = ignore_mismatches options = a_settings_source.get_values(self, ignore_mismatches=this_source_ignore_mismatches) self._overlay_value_sources_recurse(options, ignore_mismatches=this_source_ignore_mismatches) # -------------------------------------------------------------------------- def _overlay_value_sources_recurse(self, source, destination=None, prefix="", ignore_mismatches=True): if destination is None: destination = self.option_definitions for key, val in source.items(): try: sub_destination = destination for subkey in key.split("."): sub_destination = sub_destination[subkey] except KeyError: if ignore_mismatches: continue if key == subkey: raise exc.NotAnOptionError("%s is not an option" % key) raise exc.NotAnOptionError("%s subpart %s is not an option" % (key, subkey)) except TypeError: pass if isinstance(sub_destination, Namespace): self._overlay_value_sources_recurse(val, sub_destination, prefix=("%s.%s" % (prefix, key))) elif isinstance(sub_destination, Option): sub_destination.set_value(val) elif isinstance(sub_destination, Aggregation): # there is nothing to do for Aggregations at this time # it appears here anyway as a marker for future enhancements pass # -------------------------------------------------------------------------- def _walk_config_copy_values(self, source, destination, mapping_class): for key, val in source.items(): value_type = type(val) if isinstance(val, Option) or isinstance(val, Aggregation): destination[key] = val.value elif value_type == Namespace: destination[key] = d = mapping_class() self._walk_config_copy_values(val, d, mapping_class) # -------------------------------------------------------------------------- def _aggregate(self, source, base_namespace, local_namespace): aggregates_found = False for key, val in source.items(): if isinstance(val, Namespace): new_aggregates_found = self._aggregate(val, base_namespace, local_namespace[key]) aggregates_found = new_aggregates_found or aggregates_found elif isinstance(val, Aggregation): val.aggregate(base_namespace, local_namespace, self.args) aggregates_found = True # skip Options, we're only dealing with Aggregations return aggregates_found # -------------------------------------------------------------------------- @staticmethod def _option_sort(x_tuple): key, val = x_tuple if isinstance(val, Namespace): return "zzzzzzzzzzz%s" % key else: return key # -------------------------------------------------------------------------- @staticmethod def _block_password(qkey, key, value, block_password=True): if block_password and "password" in key.lower(): value = "*********" return qkey, key, value # -------------------------------------------------------------------------- def _walk_config(self, source=None, prefix="", blocked_keys=(), block_password=False): if source == None: source = self.option_definitions options_list = source.items() options_list.sort(key=ConfigurationManager._option_sort) for key, val in options_list: qualified_key = "%s%s" % (prefix, key) if qualified_key in blocked_keys: continue if isinstance(val, Option): yield self._block_password(qualified_key, key, val, block_password) if isinstance(val, Aggregation): yield qualified_key, key, val elif isinstance(val, Namespace): if qualified_key == "admin": continue yield qualified_key, key, val new_prefix = "%s%s." % (prefix, key) for xqkey, xkey, xval in self._walk_config(val, new_prefix, blocked_keys, block_password): yield xqkey, xkey, xval # -------------------------------------------------------------------------- def _get_option(self, name): source = self.option_definitions try: for sub_name in name.split("."): candidate = source[sub_name] if isinstance(candidate, Option): return candidate else: source = candidate except KeyError: pass # we need to raise the exception below in either case # of a key error or execution falling through the loop raise exc.NotAnOptionError("%s is not a known option name" % name) # -------------------------------------------------------------------------- def _get_options(self, source=None, options=None, prefix=""): if not source: source = self.option_definitions if options is None: options = [] for key, val in source.items(): if isinstance(val, Namespace): new_prefix = "%s%s." % (prefix, key) self._get_options(val, options, new_prefix) else: options.append(("%s%s" % (prefix, key), val)) return options
class ConfigurationManager(object): #-------------------------------------------------------------------------- def __init__(self, definition_source=None, values_source_list=None, argv_source=None, #use_config_files=True, use_auto_help=True, use_admin_controls=True, quit_after_admin=True, options_banned_from_help=None, app_name='', app_version='', app_description='', config_pathname='.', config_optional=True, ): """create and initialize a configman object. parameters: definition_source - a namespace or list of namespaces from which configman is to fetch the definitions of the configuration parameters. values_source_list - (optional) a hierarchical list of sources for values for the configuration parameters. As values are copied from these sources, conficting values are resolved with sources on the right getting preference over sources on the left. argv_source - if the values_source_list contains a commandline source, this value is an alternative source for actual command line arguments. Useful for testing or preprocessing command line arguments. use_auto_help - set to True if configman is to automatically set up help output for command line invocations. use_admin_controls - configman can add command line flags that it interprets independently of the app defined arguments. True enables this capability, while, False supresses it. quit_after_admin - if True and admin controls are enabled and used, call sys.exit to end the app. This is useful to stop the app from running if all that was done was to write a config file or stop after help. options_banned_from_help - a list of strings that will censor the output of help to prevent specified options from being listed in the help output. This is useful for hiding debug or secret command line arguments. app_name - assigns a name to the app. This is used in help output and as a default basename for config files. app_version - assigns a version for the app used help output. app_description - assigns a description for the app to be used in the help output. config_pathname - a hard coded path to the directory of or the full path and name of the configuration file. config_optional - a boolean indicating if a missing default config file is optional. Note: this is only for the default config file. If a config file is specified on the commandline, it _must_ exsist.""" # instead of allowing mutables as default keyword argument values... if definition_source is None: definition_source_list = [] elif (isinstance(definition_source, collections.Sequence) and not isinstance(definition_source, basestring)): definition_source_list = list(definition_source) else: definition_source_list = [definition_source] if argv_source is None: self.argv_source = sys.argv[1:] self.app_invocation_name = sys.argv[0] else: self.argv_source = argv_source self.app_invocation_name = app_name if options_banned_from_help is None: options_banned_from_help = ['application'] self.config_pathname = config_pathname self.config_optional = config_optional self.app_name = app_name self.app_version = app_version self.app_description = app_description self.args = [] # extra commandline arguments that are not switches # will be stored here. self._config = None # eventual container for DOM-like config object self.option_definitions = Namespace() self.definition_source_list = definition_source_list if values_source_list is None: # nothing set, assume defaults if use_admin_controls: values_source_list = (cm.ConfigFileFutureProxy, cm.environment, cm.command_line) else: values_source_list = (cm.environment, cm.command_line) admin_tasks_done = False self.admin_controls_list = ['help', 'admin.conf', 'admin.dump_conf', 'admin.print_conf', 'admin.migration', 'admin.strict' ] self.options_banned_from_help = options_banned_from_help if use_auto_help: self._setup_auto_help() if use_admin_controls: admin_options = self._setup_admin_options(values_source_list) self.definition_source_list.append(admin_options) # iterate through the option definitions to create the nested dict # hierarchy of all the options called 'option_definitions' for a_definition_source in self.definition_source_list: try: safe_copy_of_def_source = a_definition_source.safe_copy() except AttributeError: # apparently, the definition source was not in the form of a # Namespace object. This isn't a show stopper, but we don't # know how to make a copy of this object safely: we know from # experience that the stock copy.copy method leads to grief # as many sub-objects within an option definition source can # not be copied that way (classes, for example). # The only action we can take is to trust and continue with the # original copy of the definition source. safe_copy_of_def_source = a_definition_source def_sources.setup_definitions(safe_copy_of_def_source, self.option_definitions) if use_admin_controls: # the name of the config file needs to be loaded from the command # line prior to processing the rest of the command line options. config_filename = \ value_sources.config_filename_from_commandline(self) if ( config_filename and cm.ConfigFileFutureProxy in values_source_list ): self.option_definitions.admin.conf.default = config_filename self.values_source_list = value_sources.wrap( values_source_list, self ) known_keys = self._overlay_expand() self._check_for_mismatches(known_keys) # the app_name, app_version and app_description are to come from # if 'application' option if it is present. If it is not present, # get the app_name,et al, from parameters passed into the constructor. # if those are empty, set app_name, et al, to empty strings try: app_option = self._get_option('application') self.app_name = getattr(app_option.value, 'app_name', '') self.app_version = getattr(app_option.value, 'app_version', '') self.app_description = getattr(app_option.value, 'app_description', '') except exc.NotAnOptionError: # there is no 'application' option, continue to use the # 'app_name' from the parameters passed in, if they exist. pass if use_auto_help and self._get_option('help').value: self.output_summary() admin_tasks_done = True if use_admin_controls and self._get_option('admin.print_conf').value: self.print_conf() admin_tasks_done = True if use_admin_controls and self._get_option('admin.dump_conf').value: self.dump_conf() admin_tasks_done = True if quit_after_admin and admin_tasks_done: sys.exit() #-------------------------------------------------------------------------- @contextlib.contextmanager def context(self): """return a config as a context that calls close on every item when it goes out of scope""" config = None try: config = self.get_config() yield config finally: if config: self._walk_and_close(config) #-------------------------------------------------------------------------- def get_config(self, mapping_class=DotDictWithAcquisition): config = self._generate_config(mapping_class) if self._aggregate(self.option_definitions, config, config): # state changed, must regenerate return self._generate_config(mapping_class) else: return config #-------------------------------------------------------------------------- def output_summary(self, output_stream=sys.stdout, block_password=True): """outputs a usage tip and the list of acceptable commands. This is useful as the output of the 'help' option. parameters: output_stream - an open file-like object suitable for use as the target of a print statement block_password - a boolean driving the use of a string of * in place of the value for any object containing the substring 'passowrd' """ if self.app_name or self.app_description: print >> output_stream, 'Application:', if self.app_name: print >> output_stream, self.app_name, self.app_version if self.app_description: print >> output_stream, self.app_description if self.app_name or self.app_description: print >> output_stream, '' names_list = self.get_option_names() print >> output_stream, "usage:\n", self.app_invocation_name, \ "[OPTIONS]...", bracket_count = 0 for key in names_list: an_option = self.option_definitions[key] if an_option.is_argument: if an_option.default is None: print >> output_stream, an_option.name, else: print >> output_stream, "[ %s" % an_option.name, bracket_count += 1 print >> output_stream, ']' * bracket_count, '\n' names_list.sort() if names_list: print >> output_stream, 'OPTIONS:' pad = ' ' * 4 for name in names_list: if name in self.options_banned_from_help: continue option = self._get_option(name) line = ' ' * 2 # always start with 2 spaces if option.short_form: line += '-%s, ' % option.short_form line += '--%s' % name line += '\n' doc = option.doc if option.doc is not None else '' if doc: line += '%s%s\n' % (pad, doc) try: value = option.value type_of_value = type(value) converter_function = conv.to_string_converters[type_of_value] default = converter_function(value) except KeyError: default = option.value if default is not None: if 'password' in name.lower(): default = '*********' if name not in ('help',): # don't bother with certain dead obvious ones line += '%s(default: %s)\n' % (pad, default) print >> output_stream, line #-------------------------------------------------------------------------- def print_conf(self): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" config_file_type = self._get_option('admin.print_conf').value @contextlib.contextmanager def stdout_opener(): yield sys.stdout skip_keys = [k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_print_conf] self.write_conf(config_file_type, stdout_opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def dump_conf(self, config_pathname=None): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" if not config_pathname: config_pathname = self._get_option('admin.dump_conf').value opener = functools.partial(open, config_pathname, 'w') config_file_type = os.path.splitext(config_pathname)[1][1:] skip_keys = [k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_dump_conf] self.write_conf(config_file_type, opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def write_conf(self, config_file_type, opener, skip_keys=None): """write a configuration file to a file-like object. parameters: config_file_type - a string containing a registered file type OR a for_XXX module from the value_source package. Passing in an string that is unregistered will result in a KeyError opener - a callable object or function that returns a file like object that works as a context in a with statement.""" blocked_keys = self.admin_controls_list if skip_keys: blocked_keys.extend(skip_keys) if blocked_keys: option_defs = self.option_definitions.safe_copy() for a_blocked_key in blocked_keys: try: del option_defs[a_blocked_key] except (AttributeError, KeyError): # okay that key isn't here pass # remove empty namespaces all_keys = [k for k in option_defs.keys_breadth_first(include_dicts=True)] for key in all_keys: candidate = option_defs[key] if (isinstance(candidate, Namespace) and not len(candidate)): del option_defs[key] else: option_defs = self.option_definitions if self.option_definitions.admin.migration.default: self._migrate_options_for_acquisition(option_defs) value_sources.write(config_file_type, option_defs, opener) #-------------------------------------------------------------------------- @staticmethod def _migrate_options_for_acquisition(option_defs): """sift through the definitions looking for common keys that can be migrated to a lower level in the hierarchy. create a mapping with these characteristics: key - composed of the values from Option objects for 'name', 'default' and 'from_string' converstion function. value - a list of the the keys from the currently active option definition. this implements a reverse index of value to keys for the system of Namespaces and Options that make up the configuration manager's option definition mapping. the mapping is keyed by the option name, its default value and the from_string_converter function. If all these things are the same in two options, then it is considered that they are referring to the same option. This means that that option may be able migrate to a lower level. The mitigating factor is the number of Options with the same key name. If there is more than one with the same name, then the Option cannot migrate to a lower level. """ migration_candidates = collections.defaultdict(list) option_name_counts = collections.defaultdict(int) # cycle through all the keys for option_name in option_defs.keys_breadth_first(): # option_name is a fully qualified key: x.y.z an_option = option_defs[option_name] if isinstance(an_option, Option): name_default_converter_key = ( an_option.name, str(an_option.default), str(an_option.from_string_converter) ) if name_default_converter_key not in migration_candidates: option_name_counts[an_option.name] += 1 migration_candidates[name_default_converter_key].append( option_name ) for candidate, original_keys in migration_candidates.iteritems(): # candidate is: (option_name, option_default, option_coverter) # remove name qualifications: x.y.z --> z if ( option_name_counts[candidate[0]] == 1 and len(original_keys) > 1 ): option_defs[candidate[0]] = \ option_defs[original_keys[0]].copy() option_defs[candidate[0]].not_for_definition = True for a_key in original_keys: option_defs[a_key].comment_out = True #-------------------------------------------------------------------------- def log_config(self, logger): """write out the current configuration to a log-like object. parameters: logger - a object that implements a method called 'info' with the same semantics as the call to 'logger.info'""" logger.info("app_name: %s", self.app_name) logger.info("app_version: %s", self.app_version) logger.info("current configuration:") config = [(key, self.option_definitions[key].value) for key in self.option_definitions.keys_breadth_first() if key not in self.admin_controls_list] config.sort() for key, val in config: if 'password' in key.lower(): logger.info('%s: *********', key) else: try: logger.info('%s: %s', key, conv.to_string_converters[type(key)](val)) except KeyError: logger.info('%s: %s', key, val) #-------------------------------------------------------------------------- def get_option_names(self): """returns a list of fully qualified option names. returns: a list of strings representing the Options in the source Namespace list. Each item will be fully qualified with dot delimited Namespace names. """ return [x for x in self.option_definitions.keys_breadth_first() if isinstance(self.option_definitions[x], Option)] #-------------------------------------------------------------------------- def _overlay_expand(self): """This method overlays each of the value sources onto the default in each of the defined options. It does so using a breadth first iteration, overlaying and expanding each level of the tree in turn. As soon as no changes were made to any level, the loop breaks and the work is done. The actual action of the overlay is to take the value from the source and copy into the 'default' member of each Option object. "expansion" means converting an option value into its real type from string. The conversion is accomplished by simply calling the 'set_value' method of the Option object. If the resultant type has its own configuration options, bring those into the current namespace and then proceed to overlay/expand those. """ new_keys_discovered = True # loop control, False breaks the loop known_keys = set() # a set of keys that have been expanded while new_keys_discovered: # loop until nothing more is done # keys holds a list of all keys in the option definitons in # breadth first order using this form: [ 'x', 'y', 'z', 'x.a', # 'x.b', 'z.a', 'z.b', 'x.a.j', 'x.a.k', 'x.b.h'] keys = [x for x in self.option_definitions.keys_breadth_first()] new_keys_discovered = False # setup to break loop # overlay process: # fetch all the default values from the value sources before # applying the from string conversions for key in keys: if key not in known_keys: # skip all keys previously seen # loop through all the value sources looking for values # that match this current key. for a_value_source in self.values_source_list: try: # get all the option values from this value source val_src_dict = a_value_source.get_values( self, True ) # make sure it is in the form of a DotDict if not isinstance(val_src_dict, DotDict): val_src_dict = \ DotDictWithAcquisition(val_src_dict) # get the Option for this key opt = self.option_definitions[key] # overlay the default with the new value from # the value source. This assignment may come # via acquisition, so the key given may not have # been an exact match for what was returned. opt.default = val_src_dict[key] except KeyError, x: pass # okay, that source doesn't have this value # expansion process: # step through all the keys converting them to their proper # types and bringing in any new keys in the process for key in keys: if key not in known_keys: # skip all keys previously seen # mark this key as having been seen and processed known_keys.add(key) an_option = self.option_definitions[key] if isinstance(an_option, Aggregation): continue # aggregations are ignored # apply the from string conversion to make the real value an_option.set_value(an_option.default) # new values have been seen, don't let loop break new_keys_discovered = True try: try: # try to fetch new requirements from this value new_req = an_option.value.get_required_config() except AttributeError: new_req = an_option.value.required_config # make sure what we got as new_req is actually a # Mapping of some sort if not isinstance(new_req, collections.Mapping): # we didn't get a mapping, perhaps the option value # was a Mock object - in any case we can't try to # interpret 'new_req' as a configman requirement # collection. We must abandon processing this # option further continue # get the parent namespace current_namespace = self.option_definitions.parent(key) if current_namespace is None: # we're at the top level, use the base namespace current_namespace = self.option_definitions # add the new Options to the namespace current_namespace.update(new_req.safe_copy()) except AttributeError, x: # there are apparently no new Options to bring in from # this option's value pass
class ConfigurationManager(object): #-------------------------------------------------------------------------- def __init__( self, definition_source=None, values_source_list=None, argv_source=None, #use_config_files=True, use_auto_help=True, use_admin_controls=True, quit_after_admin=True, options_banned_from_help=None, app_name='', app_version='', app_description='', config_pathname='.', ): """create and initialize a configman object. parameters: definition_source - a namespace or list of namespaces from which configman is to fetch the definitions of the configuration parameters. values_source_list - (optional) a hierarchical list of sources for values for the configuration parameters. As values are copied from these sources, conficting values are resolved with sources on the right getting preference over sources on the left. argv_source - if the values_source_list contains a commandline source, this value is an alternative source for actual command line arguments. Useful for testing or preprocessing command line arguments. use_auto_help - set to True if configman is to automatically set up help output for command line invocations. use_admin_controls - configman can add command line flags that it interprets independently of the app defined arguments. True enables this capability, while, False supresses it. quit_after_admin - if True and admin controls are enabled and used, call sys.exit to end the app. This is useful to stop the app from running if all that was done was to write a config file or stop after help. options_banned_from_help - a list of strings that will censor the output of help to prevent specified options from being listed in the help output. This is useful for hiding debug or secret command line arguments. app_name - assigns a name to the app. This is used in help output and as a default basename for config files. app_version - assigns a version for the app used help output. app_description - assigns a description for the app to be used in the help output. config_pathname - a hard coded path to the directory of or the full path and name of the configuration file.""" # instead of allowing mutables as default keyword argument values... if definition_source is None: definition_source_list = [] elif (isinstance(definition_source, collections.Sequence) and not isinstance(definition_source, basestring)): definition_source_list = list(definition_source) else: definition_source_list = [definition_source] if argv_source is None: argv_source = sys.argv[1:] if options_banned_from_help is None: options_banned_from_help = ['admin.application'] self.config_pathname = config_pathname self.app_name = app_name self.app_version = app_version self.app_description = app_description self.args = [] # extra commandline arguments that are not switches # will be stored here. self._config = None # eventual container for DOM-like config object self.argv_source = argv_source self.option_definitions = Namespace() self.definition_source_list = definition_source_list if values_source_list is None: # nothing set, assume defaults if use_admin_controls: values_source_list = (cm.ConfigFileFutureProxy, cm.environment, cm.command_line) else: values_source_list = (cm.environment, cm.command_line) admin_tasks_done = False self.admin_controls_list = [ 'help', 'admin.conf', 'admin.dump_conf', 'admin.print_conf', 'admin.application' ] self.options_banned_from_help = options_banned_from_help if use_auto_help: self._setup_auto_help() if use_admin_controls: admin_options = self._setup_admin_options(values_source_list) self.definition_source_list.append(admin_options) # iterate through the option definitions to create the nested dict # hierarchy of all the options called 'option_definitions' for a_definition_source in self.definition_source_list: def_sources.setup_definitions(a_definition_source, self.option_definitions) if use_admin_controls: # some admin options need to be loaded from the command line # prior to processing the rest of the command line options. admin_options = value_sources.get_admin_options_from_command_line( self) # integrate the admin_options with 'option_definitions' self._overlay_value_sources_recurse(source=admin_options, ignore_mismatches=True) self.values_source_list = value_sources.wrap(values_source_list, self) # first pass to get classes & config path - ignore bad options self._overlay_value_sources(ignore_mismatches=True) # walk tree expanding class options self._walk_expanding_class_options() # the app_name, app_version and app_description are to come from # if 'admin.application' option if it is present. If it is not present, # get the app_name,et al, from parameters passed into the constructor. # if those are empty, set app_name, et al, to empty strings try: app_option = self._get_option('admin.application') self.app_name = getattr(app_option.value, 'app_name', '') self.app_version = getattr(app_option.value, 'app_version', '') self.app_description = getattr(app_option.value, 'app_description', '') except exc.NotAnOptionError: # there is no 'admin.application' option, continue to use the # 'app_name' from the parameters passed in, if they exist. pass # second pass to include config file values - ignore bad options self._overlay_value_sources(ignore_mismatches=True) # walk tree expanding class options self._walk_expanding_class_options() # third pass to get values - complain about bad options self._overlay_value_sources(ignore_mismatches=False) if use_auto_help and self._get_option('help').value: self.output_summary() admin_tasks_done = True if (use_admin_controls and self._get_option('admin.print_conf').value): self.print_conf() admin_tasks_done = True if (use_admin_controls and self._get_option('admin.dump_conf').value): self.dump_conf() admin_tasks_done = True if quit_after_admin and admin_tasks_done: sys.exit() #-------------------------------------------------------------------------- @contextlib.contextmanager def context(self): """return a config as a context that calls close on every item when it goes out of scope""" config = None try: config = self.get_config() yield config finally: if config: self._walk_and_close(config) #-------------------------------------------------------------------------- def get_config(self, mapping_class=DotDictWithAcquisition): config = self._generate_config(mapping_class) if self._aggregate(self.option_definitions, config, config): # state changed, must regenerate return self._generate_config(mapping_class) else: return config #-------------------------------------------------------------------------- def output_summary(self, output_stream=sys.stdout, block_password=True): """outputs a usage tip and the list of acceptable commands. This is useful as the output of the 'help' option. parameters: output_stream - an open file-like object suitable for use as the target of a print statement block_password - a boolean driving the use of a string of * in place of the value for any object containing the substring 'passowrd' """ if self.app_name or self.app_description: print >> output_stream, 'Application:', if self.app_name: print >> output_stream, self.app_name, self.app_version if self.app_description: print >> output_stream, self.app_description if self.app_name or self.app_description: print >> output_stream, '' names_list = self.get_option_names() names_list.sort() if names_list: print >> output_stream, 'Options:' for name in names_list: if name in self.options_banned_from_help: continue option = self._get_option(name) line = ' ' * 2 # always start with 2 spaces if option.short_form: line += '-%s, ' % option.short_form line += '--%s' % name line = line.ljust(30) # seems to the common practise doc = option.doc if option.doc is not None else '' try: value = option.value type_of_value = type(value) converter_function = conv.to_string_converters[type_of_value] default = converter_function(value) except KeyError: default = option.value if default is not None: if 'password' in name.lower(): default = '*********' if doc: doc += ' ' if name not in ('help', ): # don't bother with certain dead obvious ones doc += '(default: %s)' % default line += doc print >> output_stream, line #-------------------------------------------------------------------------- def print_conf(self): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" config_file_type = self._get_option('admin.print_conf').value @contextlib.contextmanager def stdout_opener(): yield sys.stdout skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_print_conf ] self.write_conf(config_file_type, stdout_opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def dump_conf(self, config_pathname=None): """write a config file to the pathname specified in the parameter. The file extention determines the type of file written and must match a registered type. parameters: config_pathname - the full path and filename of the target config file.""" if not config_pathname: config_pathname = self._get_option('admin.dump_conf').value opener = functools.partial(open, config_pathname, 'w') config_file_type = os.path.splitext(config_pathname)[1][1:] skip_keys = [ k for (k, v) in self.option_definitions.iteritems() if isinstance(v, Option) and v.exclude_from_dump_conf ] self.write_conf(config_file_type, opener, skip_keys=skip_keys) #-------------------------------------------------------------------------- def write_conf(self, config_file_type, opener, skip_keys=None): """write a configuration file to a file-like object. parameters: config_file_type - a string containing a registered file type OR a for_XXX module from the value_source package. Passing in an string that is unregistered will result in a KeyError opener - a callable object or function that returns a file like object that works as a context in a with statement.""" blocked_keys = self.admin_controls_list if skip_keys: blocked_keys.extend(skip_keys) option_iterator = functools.partial(self._walk_config, blocked_keys=blocked_keys) with opener() as config_fp: value_sources.write(config_file_type, option_iterator, config_fp) #-------------------------------------------------------------------------- def log_config(self, logger): """write out the current configuration to a log-like object. parameters: logger - a object that implements a method called 'info' with the same semantics as the call to 'logger.info'""" logger.info("app_name: %s", self.app_name) logger.info("app_version: %s", self.app_version) logger.info("current configuration:") config = [ (qkey, val.value) for qkey, key, val in self._walk_config(self.option_definitions) if qkey not in self.admin_controls_list and not isinstance(val, Namespace) ] config.sort() for key, val in config: if 'password' in key.lower(): logger.info('%s: *********', key) else: try: logger.info('%s: %s', key, conv.to_string_converters[type(key)](val)) except KeyError: logger.info('%s: %s', key, val) #-------------------------------------------------------------------------- def get_option_names(self, source=None, names=None, prefix=''): """returns a list of fully qualified option names. parameters: source - a sequence of Namespace of Options, usually not specified, If not specified, the function will default to using the internal list of Option definitions. names - a list to start with for appending the lsit Option names. If ommited, the function will start with an empty list. returns: a list of strings representing the Options in the source Namespace list. Each item will be fully qualified with dot delimited Namespace names. """ if not source: source = self.option_definitions if names is None: names = [] for key, val in source.items(): if isinstance(val, Namespace): new_prefix = '%s%s.' % (prefix, key) self.get_option_names(val, names, new_prefix) elif isinstance(val, Option): names.append("%s%s" % (prefix, key)) # skip aggregations, we want only Options return names #-------------------------------------------------------------------------- @staticmethod def _walk_and_close(a_dict): for val in a_dict.itervalues(): if isinstance(val, collections.Mapping): ConfigurationManager._walk_and_close(val) if hasattr(val, 'close') and not inspect.isclass(val): val.close() #-------------------------------------------------------------------------- def _generate_config(self, mapping_class): """This routine generates a copy of the DotDict based config""" config = mapping_class() self._walk_config_copy_values(self.option_definitions, config, mapping_class) return config #-------------------------------------------------------------------------- def _walk_expanding_class_options(self, source_namespace=None, parent_namespace=None): if source_namespace is None: source_namespace = self.option_definitions expanded_keys = [] expansions_were_done = True while expansions_were_done: expansions_were_done = False # can't use iteritems in loop, we're changing the dict for key, val in source_namespace.items(): if isinstance(val, Namespace): self._walk_expanding_class_options( source_namespace=val, parent_namespace=source_namespace) elif (key not in expanded_keys and (inspect.isclass(val.value) or inspect.ismodule(val.value))): expanded_keys.append(key) expansions_were_done = True if key == 'application': target_namespace = parent_namespace else: target_namespace = source_namespace try: for o_key, o_val in \ val.value.get_required_config().iteritems(): target_namespace.__setattr__( o_key, copy.deepcopy(o_val)) except AttributeError: pass # there are no required_options for this class else: pass # don't need to touch other types of Options self._overlay_value_sources(ignore_mismatches=True) #-------------------------------------------------------------------------- def _setup_auto_help(self): help_option = Option(name='help', doc='print this', default=False) self.definition_source_list.append({'help': help_option}) #-------------------------------------------------------------------------- def _get_config_pathname(self): if os.path.isdir(self.config_pathname): # we've got a path with no file name at the end # use the appname as the file name and default to an 'ini' # config file type if self.app_name: return os.path.join(self.config_pathname, '%s.ini' % self.app_name) else: # there is no app_name yet # we'll punt and use 'config' return os.path.join(self.config_pathname, 'config.ini') return self.config_pathname #-------------------------------------------------------------------------- def _setup_admin_options(self, values_source_list): base_namespace = Namespace() base_namespace.admin = admin = Namespace() admin.add_option( name='print_conf', default=None, doc='write current config to stdout (%s)' % ', '.join(value_sources.file_extension_dispatch.keys())) admin.add_option( name='dump_conf', default='', doc='a pathname to which to write the current config', ) # only offer the config file admin options if they've been requested in # the values source list if ConfigFileFutureProxy in values_source_list: default_config_pathname = self._get_config_pathname() admin.add_option( name='conf', default=default_config_pathname, doc='the pathname of the config file ' '(path/filename)', ) return base_namespace #-------------------------------------------------------------------------- def _overlay_value_sources(self, ignore_mismatches=True): for a_settings_source in self.values_source_list: try: this_source_ignore_mismatches = ( ignore_mismatches or a_settings_source.always_ignore_mismatches) except AttributeError: # the settings source doesn't have the concept of always # ignoring mismatches, so the original value of # ignore_mismatches stands this_source_ignore_mismatches = ignore_mismatches options = a_settings_source.get_values( self, ignore_mismatches=this_source_ignore_mismatches) self._overlay_value_sources_recurse( options, ignore_mismatches=this_source_ignore_mismatches) #-------------------------------------------------------------------------- def _overlay_value_sources_recurse(self, source, destination=None, prefix='', ignore_mismatches=True): if destination is None: destination = self.option_definitions for key, val in source.items(): try: sub_destination = destination for subkey in key.split('.'): sub_destination = sub_destination[subkey] except KeyError: if ignore_mismatches: continue if key == subkey: raise exc.NotAnOptionError('%s is not an option' % key) raise exc.NotAnOptionError('%s subpart %s is not an option' % (key, subkey)) except TypeError: pass if isinstance(sub_destination, Namespace): self._overlay_value_sources_recurse(val, sub_destination, prefix=('%s.%s' % (prefix, key))) elif isinstance(sub_destination, Option): sub_destination.set_value(val) elif isinstance(sub_destination, Aggregation): # there is nothing to do for Aggregations at this time # it appears here anyway as a marker for future enhancements pass #-------------------------------------------------------------------------- def _walk_config_copy_values(self, source, destination, mapping_class): for key, val in source.items(): value_type = type(val) if isinstance(val, Option) or isinstance(val, Aggregation): destination[key] = val.value elif value_type == Namespace: destination[key] = d = mapping_class() self._walk_config_copy_values(val, d, mapping_class) #-------------------------------------------------------------------------- def _aggregate(self, source, base_namespace, local_namespace): aggregates_found = False for key, val in source.items(): if isinstance(val, Namespace): new_aggregates_found = self._aggregate(val, base_namespace, local_namespace[key]) aggregates_found = new_aggregates_found or aggregates_found elif isinstance(val, Aggregation): val.aggregate(base_namespace, local_namespace, self.args) aggregates_found = True # skip Options, we're only dealing with Aggregations return aggregates_found #-------------------------------------------------------------------------- @staticmethod def _option_sort(x_tuple): key, val = x_tuple if isinstance(val, Namespace): return 'zzzzzzzzzzz%s' % key else: return key #-------------------------------------------------------------------------- @staticmethod def _block_password(qkey, key, value, block_password=True): if block_password and 'password' in key.lower(): value = '*********' return qkey, key, value #-------------------------------------------------------------------------- def _walk_config(self, source=None, prefix='', blocked_keys=(), block_password=False): if source == None: source = self.option_definitions options_list = source.items() options_list.sort(key=ConfigurationManager._option_sort) for key, val in options_list: qualified_key = '%s%s' % (prefix, key) if qualified_key in blocked_keys: continue if isinstance(val, Option): yield self._block_password(qualified_key, key, val, block_password) if isinstance(val, Aggregation): yield qualified_key, key, val elif isinstance(val, Namespace): if qualified_key == 'admin': continue yield qualified_key, key, val new_prefix = '%s%s.' % (prefix, key) for xqkey, xkey, xval in self._walk_config( val, new_prefix, blocked_keys, block_password): yield xqkey, xkey, xval #-------------------------------------------------------------------------- def _get_option(self, name): source = self.option_definitions try: for sub_name in name.split('.'): candidate = source[sub_name] if isinstance(candidate, Option): return candidate else: source = candidate except KeyError: pass # we need to raise the exception below in either case # of a key error or execution falling through the loop raise exc.NotAnOptionError('%s is not a known option name' % name) #-------------------------------------------------------------------------- def _get_options(self, source=None, options=None, prefix=''): if not source: source = self.option_definitions if options is None: options = [] for key, val in source.items(): if isinstance(val, Namespace): new_prefix = '%s%s.' % (prefix, key) self._get_options(val, options, new_prefix) else: options.append(("%s%s" % (prefix, key), val)) return options