def plugin_list(value): """Turn a space-delimited series of plugin names into a ListAndAll of Plugins. Never return the core plugin. """ if not isinstance(value, basestring): # Probably can't happen raise SchemaError('"%s" is neither * nor a whitespace-delimited list ' 'of plugin names.' % (value,), None) plugins = all_plugins_but_core() names = value.strip().split() is_all = names == ['*'] if is_all: names = plugins.keys() try: ret = ListAndAll([plugins[name] for name in names]) ret.is_all = is_all return ret except KeyError: raise SchemaError('Never heard of plugin "%s". I\'ve heard of ' 'these: %s.' % (name, ', '.join(plugins.keys())), None)
def plugin_list(value): """Turn a space-delimited series of plugin names into a ListAndAll of Plugins. Never return the core plugin. """ if not isinstance(value, basestring): # Probably can't happen raise SchemaError( '"%s" is neither * nor a whitespace-delimited list ' 'of plugin names.' % (value, ), None) plugins = all_plugins_but_core() names = value.strip().split() is_all = names == ['*'] if is_all: names = plugins.keys() try: ret = ListAndAll([plugins[name] for name in names]) ret.is_all = is_all return ret except KeyError: raise SchemaError( 'Never heard of plugin "%s". I\'ve heard of ' 'these: %s.' % (name, ', '.join(plugins.keys())), None)
def __init__(self, input, relative_to=None): """Pull in and validate a config file. :arg input: A string or dict from which to populate the config :arg relative_to: The dir relative to which to interpret relative paths Raise ConfigError if the configuration is invalid. """ schema = Schema({ 'DXR': { Optional('temp_folder', default=abspath('dxr-temp-{tree}')): AbsPath, Optional('default_tree', default=None): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('generated_date', default=datetime.utcnow() .strftime("%a, %d %b %Y %H:%M:%S +0000")): basestring, Optional('log_folder', default=abspath('dxr-logs-{tree}')): AbsPath, Optional('workers', default=if_raises(NotImplementedError, cpu_count, 1)): WORKERS_VALIDATOR, Optional('skip_stages', default=[]): WhitespaceList, Optional('www_root', default=''): Use(lambda v: v.rstrip('/')), Optional('google_analytics_key', default=''): basestring, Optional('es_hosts', default='http://127.0.0.1:9200/'): WhitespaceList, # A semi-random name, having the tree name and format version in it. Optional('es_index', default='dxr_{format}_{tree}_{unique}'): basestring, Optional('es_alias', default='dxr_{format}_{tree}'): basestring, Optional('es_catalog_index', default='dxr_catalog'): basestring, Optional('es_catalog_replicas', default=1): Use(int, error='"es_catalog_replicas" must be an integer.'), Optional('max_thumbnail_size', default=20000): And(Use(int), lambda v: v >= 0, error='"max_thumbnail_size" must be a non-negative ' 'integer.'), Optional('es_indexing_timeout', default=60): And(Use(int), lambda v: v >= 0, error='"es_indexing_timeout" must be a non-negative ' 'integer.'), Optional('es_indexing_retries', default=0): And(Use(int), lambda v: v >= 0, error='"es_indexing_retries" must be a non-negative ' 'integer.'), Optional('es_refresh_interval', default=60): Use(int, error='"es_refresh_interval" must be an integer.') }, basestring: dict }) # Parse the ini into nested dicts: config_obj = ConfigObj(input.splitlines() if isinstance(input, basestring) else input, list_values=False) if not relative_to: relative_to = getcwd() with cd(relative_to): try: config = schema.validate(config_obj.dict()) except SchemaError as exc: raise ConfigError(exc.code, ['DXR']) self._section = config['DXR'] # Normalize enabled_plugins: if self.enabled_plugins.is_all: # Then explicitly enable anything that isn't explicitly # disabled: self._section['enabled_plugins'] = [ p for p in all_plugins_but_core().values() if p not in self.disabled_plugins] # Now that enabled_plugins and the other keys that TreeConfig # depends on are filled out, make some TreeConfigs: self.trees = OrderedDict() # name -> TreeConfig for section in config_obj.sections: if section != 'DXR': try: self.trees[section] = TreeConfig(section, config[section], config_obj[section].sections, self) except SchemaError as exc: raise ConfigError(exc.code, [section]) # Make sure default_tree is defined: if not self.default_tree: self._section['default_tree'] = first(self.trees.iterkeys()) # These aren't intended for actual use; they're just to influence # enabled_plugins of trees, and now we're done with them: del self._section['enabled_plugins'] del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config): """Fix up settings that depend on the [DXR] section or have inter-setting dependencies. (schema can't do multi-setting validation yet, and configobj can't do cross-section interpolation.) Add a ``config`` attr to trees as a shortcut back to the [DXR] section and a ``name`` attr to save cumbersome tuple unpacks in callers. """ self.config = config self.name = name schema = Schema({ Optional('build_command', default='make -j {workers}'): basestring, Optional('clean_command', default='make clean'): basestring, Optional('description', default=''): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('es_index', default=config.es_index): basestring, Optional('es_shards', default=5): Use(int, error='"es_shards" must be an integer.'), Optional('ignore_patterns', default=['.hg', '.git', 'CVS', '.svn', '.bzr', '.deps', '.libs', '.DS_Store', '.nfs*', '*~', '._*']): WhitespaceList, Optional('object_folder', default=None): AbsPath, 'source_folder': AbsPath, Optional('source_encoding', default='utf-8'): basestring, Optional('temp_folder', default=None): AbsPath, Optional('p4web_url', default='http://p4web/'): basestring, Optional('workers', default=None): WORKERS_VALIDATOR, Optional(basestring): dict}) tree = schema.validate(unvalidated_tree) if tree['temp_folder'] is None: tree['temp_folder'] = config.temp_folder if tree['object_folder'] is None: tree['object_folder'] = tree['source_folder'] if tree['workers'] is None: tree['workers'] = config.workers # Convert enabled_plugins to a list of plugins: if tree['disabled_plugins'].is_all: # * doesn't really mean "all" in a tree. It means "everything the # [DXR] section enabled". tree['disabled_plugins'] = config.enabled_plugins else: # Add anything globally disabled to our local disabled list: tree['disabled_plugins'].extend(p for p in config.disabled_plugins if p not in tree['disabled_plugins']) if tree['enabled_plugins'].is_all: tree['enabled_plugins'] = [p for p in config.enabled_plugins if p not in tree['disabled_plugins']] tree['enabled_plugins'].insert(0, core_plugin()) # Split ignores into paths and filenames: tree['ignore_paths'] = [i for i in tree['ignore_patterns'] if i.startswith('/')] tree['ignore_filenames'] = [i for i in tree['ignore_patterns'] if not i.startswith('/')] # Delete misleading, useless, or raw values people shouldn't use: del tree['ignore_patterns'] del tree['disabled_plugins'] # Validate plugin config: enableds_with_all_optional_config = set( p for p in tree['enabled_plugins'] if all(isinstance(k, Optional) for k in p.config_schema.iterkeys())) plugin_schema = Schema(merge( dict((Optional(name) if plugin in enableds_with_all_optional_config or plugin not in tree['enabled_plugins'] else name, plugin.config_schema) for name, plugin in all_plugins_but_core().iteritems()), # And whatever isn't a plugin section, that we don't care about: {object: object})) # Insert empty missing sections for enabled plugins with entirely # optional config so their defaults get filled in. (Don't insert them # if the plugin has any required options; then we wouldn't produce the # proper error message about the section being absent.) for plugin in enableds_with_all_optional_config: tree.setdefault(plugin.name, {}) tree = plugin_schema.validate(tree) super(TreeConfig, self).__init__(tree)
def __init__(self, input, relative_to=None): """Pull in and validate a config file. :arg input: A string or dict from which to populate the config :arg relative_to: The dir relative to which to interpret relative paths Raise ConfigError if the configuration is invalid. """ schema = Schema({ 'DXR': { Optional('temp_folder', default=abspath('dxr-temp-{tree}')): AbsPath, Optional('default_tree', default=None): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('generated_date', default=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")): basestring, Optional('log_folder', default=abspath('dxr-logs-{tree}')): AbsPath, Optional('workers', default=if_raises(NotImplementedError, cpu_count, 1)): And(Use(int), lambda v: v >= 0, error='"workers" must be a non-negative integer.'), Optional('skip_stages', default=[]): WhitespaceList, Optional('www_root', default=''): Use(lambda v: v.rstrip('/')), Optional('google_analytics_key', default=''): basestring, Optional('es_hosts', default='http://127.0.0.1:9200/'): WhitespaceList, # A semi-random name, having the tree name and format version in it. Optional('es_index', default='dxr_{format}_{tree}_{unique}'): basestring, Optional('es_alias', default='dxr_{format}_{tree}'): basestring, Optional('es_catalog_index', default='dxr_catalog'): basestring, Optional('es_catalog_replicas', default=1): Use(int, error='"es_catalog_replicas" must be an integer.'), Optional('max_thumbnail_size', default=20000): And(Use(int), lambda v: v >= 0, error='"max_thumbnail_size" must be a non-negative ' 'integer.'), Optional('es_indexing_timeout', default=60): And(Use(int), lambda v: v >= 0, error='"es_indexing_timeout" must be a non-negative ' 'integer.'), Optional('es_indexing_retries', default=0): And(Use(int), lambda v: v >= 0, error='"es_indexing_retries" must be a non-negative ' 'integer.'), Optional('es_refresh_interval', default=60): Use(int, error='"es_refresh_interval" must be an integer.') }, basestring: dict }) # Parse the ini into nested dicts: config_obj = ConfigObj( input.splitlines() if isinstance(input, basestring) else input, list_values=False) if not relative_to: relative_to = getcwd() with cd(relative_to): try: config = schema.validate(config_obj.dict()) except SchemaError as exc: raise ConfigError(exc.code, ['DXR']) self._section = config['DXR'] # Normalize enabled_plugins: if self.enabled_plugins.is_all: # Then explicitly enable anything that isn't explicitly # disabled: self._section['enabled_plugins'] = [ p for p in all_plugins_but_core().values() if p not in self.disabled_plugins ] # Now that enabled_plugins and the other keys that TreeConfig # depends on are filled out, make some TreeConfigs: self.trees = OrderedDict() # name -> TreeConfig for section in config_obj.sections: if section != 'DXR': try: self.trees[section] = TreeConfig( section, config[section], config_obj[section].sections, self) except SchemaError as exc: raise ConfigError(exc.code, [section]) # Make sure default_tree is defined: if not self.default_tree: self._section['default_tree'] = first(self.trees.iterkeys()) # These aren't intended for actual use; they're just to influence # enabled_plugins of trees, and now we're done with them: del self._section['enabled_plugins'] del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config): """Fix up settings that depend on the [DXR] section or have inter-setting dependencies. (schema can't do multi-setting validation yet, and configobj can't do cross-section interpolation.) Add a ``config`` attr to trees as a shortcut back to the [DXR] section and a ``name`` attr to save cumbersome tuple unpacks in callers. """ self.config = config self.name = name schema = Schema({ Optional('build_command', default='make -j {workers}'): basestring, Optional('clean_command', default='make clean'): basestring, Optional('description', default=''): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('es_index', default=config.es_index): basestring, Optional('es_shards', default=5): Use(int, error='"es_shards" must be an integer.'), Optional('ignore_patterns', default=[ '.hg', '.git', 'CVS', '.svn', '.bzr', '.deps', '.libs', '.DS_Store', '.nfs*', '*~', '._*' ]): WhitespaceList, Optional('object_folder', default=None): AbsPath, 'source_folder': AbsPath, Optional('source_encoding', default='utf-8'): basestring, Optional('temp_folder', default=None): AbsPath, Optional('p4web_url', default='http://p4web/'): basestring, Optional(basestring): dict }) tree = schema.validate(unvalidated_tree) if tree['temp_folder'] is None: tree['temp_folder'] = config.temp_folder if tree['object_folder'] is None: tree['object_folder'] = tree['source_folder'] # Convert enabled_plugins to a list of plugins: if tree['disabled_plugins'].is_all: # * doesn't really mean "all" in a tree. It means "everything the # [DXR] section enabled". tree['disabled_plugins'] = config.enabled_plugins else: # Add anything globally disabled to our local disabled list: tree['disabled_plugins'].extend( p for p in config.disabled_plugins if p not in tree['disabled_plugins']) if tree['enabled_plugins'].is_all: tree['enabled_plugins'] = [ p for p in config.enabled_plugins if p not in tree['disabled_plugins'] ] tree['enabled_plugins'].insert(0, core_plugin()) # Split ignores into paths and filenames: tree['ignore_paths'] = [ i for i in tree['ignore_patterns'] if i.startswith('/') ] tree['ignore_filenames'] = [ i for i in tree['ignore_patterns'] if not i.startswith('/') ] # Delete misleading, useless, or raw values people shouldn't use: del tree['ignore_patterns'] del tree['disabled_plugins'] # Validate plugin config: enableds_with_all_optional_config = set( p for p in tree['enabled_plugins'] if all( isinstance(k, Optional) for k in p.config_schema.iterkeys())) plugin_schema = Schema( merge( dict((Optional(name ) if plugin in enableds_with_all_optional_config or plugin not in tree['enabled_plugins'] else name, plugin.config_schema) for name, plugin in all_plugins_but_core().iteritems()), # And whatever isn't a plugin section, that we don't care about: {object: object})) # Insert empty missing sections for enabled plugins with entirely # optional config so their defaults get filled in. (Don't insert them # if the plugin has any required options; then we wouldn't produce the # proper error message about the section being absent.) for plugin in enableds_with_all_optional_config: tree.setdefault(plugin.name, {}) tree = plugin_schema.validate(tree) super(TreeConfig, self).__init__(tree)