def plugin_list(value):
"""Turn a space-delimited series of plugin names into a ListAndAll of
Plugins.
Never return the core plugin.
"""
if not isinstance(value, basestring):
# Probably can't happen
raise SchemaError('"%s" is neither * nor a whitespace-delimited list '
'of plugin names.' % (value,), None)
plugins = all_plugins_but_core()
names = value.strip().split()
is_all = names == ['*']
if is_all:
names = plugins.keys()
try:
ret = ListAndAll([plugins[name] for name in names])
ret.is_all = is_all
return ret
except KeyError:
raise SchemaError('Never heard of plugin "%s". I\'ve heard of '
'these: %s.' % (name, ', '.join(plugins.keys())),
None)
def plugin_list(value):
"""Turn a space-delimited series of plugin names into a ListAndAll of
Plugins.
Never return the core plugin.
"""
if not isinstance(value, basestring):
# Probably can't happen
raise SchemaError(
'"%s" is neither * nor a whitespace-delimited list '
'of plugin names.' % (value, ), None)
plugins = all_plugins_but_core()
names = value.strip().split()
is_all = names == ['*']
if is_all:
names = plugins.keys()
try:
ret = ListAndAll([plugins[name] for name in names])
ret.is_all = is_all
return ret
except KeyError:
raise SchemaError(
'Never heard of plugin "%s". I\'ve heard of '
'these: %s.' % (name, ', '.join(plugins.keys())), None)
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('generated_date',
default=datetime.utcnow()
.strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers', default=if_raises(NotImplementedError,
cpu_count,
1)):
WORKERS_VALIDATOR,
Optional('skip_stages', default=[]): WhitespaceList,
Optional('www_root', default=''): Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''): basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_indexing_retries', default=0):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_retries" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_refresh_interval" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(input.splitlines() if isinstance(input,
basestring)
else input,
list_values=False)
if not relative_to:
relative_to = getcwd()
with cd(relative_to):
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins_but_core().values()
if p not in self.disabled_plugins]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(section,
config[section],
config_obj[section].sections,
self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config):
"""Fix up settings that depend on the [DXR] section or have
inter-setting dependencies. (schema can't do multi-setting validation
yet, and configobj can't do cross-section interpolation.)
Add a ``config`` attr to trees as a shortcut back to the [DXR] section
and a ``name`` attr to save cumbersome tuple unpacks in callers.
"""
self.config = config
self.name = name
schema = Schema({
Optional('build_command', default='make -j {workers}'): basestring,
Optional('clean_command', default='make clean'): basestring,
Optional('description', default=''): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('es_index', default=config.es_index): basestring,
Optional('es_shards', default=5):
Use(int, error='"es_shards" must be an integer.'),
Optional('ignore_patterns',
default=['.hg', '.git', 'CVS', '.svn', '.bzr',
'.deps', '.libs', '.DS_Store', '.nfs*', '*~',
'._*']): WhitespaceList,
Optional('object_folder', default=None): AbsPath,
'source_folder': AbsPath,
Optional('source_encoding', default='utf-8'): basestring,
Optional('temp_folder', default=None): AbsPath,
Optional('p4web_url', default='http://p4web/'): basestring,
Optional('workers', default=None): WORKERS_VALIDATOR,
Optional(basestring): dict})
tree = schema.validate(unvalidated_tree)
if tree['temp_folder'] is None:
tree['temp_folder'] = config.temp_folder
if tree['object_folder'] is None:
tree['object_folder'] = tree['source_folder']
if tree['workers'] is None:
tree['workers'] = config.workers
# Convert enabled_plugins to a list of plugins:
if tree['disabled_plugins'].is_all:
# * doesn't really mean "all" in a tree. It means "everything the
# [DXR] section enabled".
tree['disabled_plugins'] = config.enabled_plugins
else:
# Add anything globally disabled to our local disabled list:
tree['disabled_plugins'].extend(p for p in config.disabled_plugins
if p not in
tree['disabled_plugins'])
if tree['enabled_plugins'].is_all:
tree['enabled_plugins'] = [p for p in config.enabled_plugins
if p not in tree['disabled_plugins']]
tree['enabled_plugins'].insert(0, core_plugin())
# Split ignores into paths and filenames:
tree['ignore_paths'] = [i for i in tree['ignore_patterns']
if i.startswith('/')]
tree['ignore_filenames'] = [i for i in tree['ignore_patterns']
if not i.startswith('/')]
# Delete misleading, useless, or raw values people shouldn't use:
del tree['ignore_patterns']
del tree['disabled_plugins']
# Validate plugin config:
enableds_with_all_optional_config = set(
p for p in tree['enabled_plugins']
if all(isinstance(k, Optional) for k in p.config_schema.iterkeys()))
plugin_schema = Schema(merge(
dict((Optional(name) if plugin in enableds_with_all_optional_config
or plugin not in tree['enabled_plugins']
else name,
plugin.config_schema)
for name, plugin in all_plugins_but_core().iteritems()),
# And whatever isn't a plugin section, that we don't care about:
{object: object}))
# Insert empty missing sections for enabled plugins with entirely
# optional config so their defaults get filled in. (Don't insert them
# if the plugin has any required options; then we wouldn't produce the
# proper error message about the section being absent.)
for plugin in enableds_with_all_optional_config:
tree.setdefault(plugin.name, {})
tree = plugin_schema.validate(tree)
super(TreeConfig, self).__init__(tree)
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None):
basestring,
Optional('disabled_plugins', default=plugin_list('')):
Plugins,
Optional('enabled_plugins', default=plugin_list('*')):
Plugins,
Optional('generated_date',
default=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers',
default=if_raises(NotImplementedError, cpu_count, 1)):
And(Use(int),
lambda v: v >= 0,
error='"workers" must be a non-negative integer.'),
Optional('skip_stages', default=[]):
WhitespaceList,
Optional('www_root', default=''):
Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''):
basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_indexing_retries', default=0):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_retries" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_refresh_interval" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(
input.splitlines() if isinstance(input, basestring) else input,
list_values=False)
if not relative_to:
relative_to = getcwd()
with cd(relative_to):
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins_but_core().values()
if p not in self.disabled_plugins
]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(
section, config[section],
config_obj[section].sections, self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config):
"""Fix up settings that depend on the [DXR] section or have
inter-setting dependencies. (schema can't do multi-setting validation
yet, and configobj can't do cross-section interpolation.)
Add a ``config`` attr to trees as a shortcut back to the [DXR] section
and a ``name`` attr to save cumbersome tuple unpacks in callers.
"""
self.config = config
self.name = name
schema = Schema({
Optional('build_command', default='make -j {workers}'):
basestring,
Optional('clean_command', default='make clean'):
basestring,
Optional('description', default=''):
basestring,
Optional('disabled_plugins', default=plugin_list('')):
Plugins,
Optional('enabled_plugins', default=plugin_list('*')):
Plugins,
Optional('es_index', default=config.es_index):
basestring,
Optional('es_shards', default=5):
Use(int, error='"es_shards" must be an integer.'),
Optional('ignore_patterns',
default=[
'.hg', '.git', 'CVS', '.svn', '.bzr', '.deps', '.libs', '.DS_Store', '.nfs*', '*~', '._*'
]):
WhitespaceList,
Optional('object_folder', default=None):
AbsPath,
'source_folder':
AbsPath,
Optional('source_encoding', default='utf-8'):
basestring,
Optional('temp_folder', default=None):
AbsPath,
Optional('p4web_url', default='http://p4web/'):
basestring,
Optional(basestring):
dict
})
tree = schema.validate(unvalidated_tree)
if tree['temp_folder'] is None:
tree['temp_folder'] = config.temp_folder
if tree['object_folder'] is None:
tree['object_folder'] = tree['source_folder']
# Convert enabled_plugins to a list of plugins:
if tree['disabled_plugins'].is_all:
# * doesn't really mean "all" in a tree. It means "everything the
# [DXR] section enabled".
tree['disabled_plugins'] = config.enabled_plugins
else:
# Add anything globally disabled to our local disabled list:
tree['disabled_plugins'].extend(
p for p in config.disabled_plugins
if p not in tree['disabled_plugins'])
if tree['enabled_plugins'].is_all:
tree['enabled_plugins'] = [
p for p in config.enabled_plugins
if p not in tree['disabled_plugins']
]
tree['enabled_plugins'].insert(0, core_plugin())
# Split ignores into paths and filenames:
tree['ignore_paths'] = [
i for i in tree['ignore_patterns'] if i.startswith('/')
]
tree['ignore_filenames'] = [
i for i in tree['ignore_patterns'] if not i.startswith('/')
]
# Delete misleading, useless, or raw values people shouldn't use:
del tree['ignore_patterns']
del tree['disabled_plugins']
# Validate plugin config:
enableds_with_all_optional_config = set(
p for p in tree['enabled_plugins'] if all(
isinstance(k, Optional) for k in p.config_schema.iterkeys()))
plugin_schema = Schema(
merge(
dict((Optional(name
) if plugin in enableds_with_all_optional_config
or plugin not in tree['enabled_plugins'] else name,
plugin.config_schema)
for name, plugin in all_plugins_but_core().iteritems()),
# And whatever isn't a plugin section, that we don't care about:
{object: object}))
# Insert empty missing sections for enabled plugins with entirely
# optional config so their defaults get filled in. (Don't insert them
# if the plugin has any required options; then we wouldn't produce the
# proper error message about the section being absent.)
for plugin in enableds_with_all_optional_config:
tree.setdefault(plugin.name, {})
tree = plugin_schema.validate(tree)
super(TreeConfig, self).__init__(tree)
