def test_construction():
"""If a plugin is a plain module, make sure it is automatically promoted to
a Plugin via recognition of symbol naming conventions."""
plugin = all_plugins()['urllink']
mocked_tree = None # This will probably have to improve at some point.
mocked_vcs = None
ok_(isinstance(plugin.tree_to_index('urllink', mocked_tree, mocked_vcs).file_to_index('/foo/bar', ''),
urllink.FileToIndex))
def ref_class(plugin, id):
"""Return the subclass of Ref identified by a combination of
plugin and class ID."""
plugins = all_plugins()
try:
return plugins[plugin].refs[id]
except KeyError:
warn('Ref subclass from plugin %s with ID %s was referenced '
'in the index but not found in the current '
'implementation. Ignored.' % (plugin, id))
def ref_class(plugin, id):
"""Return the subclass of Ref identified by a combination of
plugin and class ID."""
plugins = all_plugins()
try:
return plugins[plugin].refs[id]
except KeyError:
warn('Ref subclass from plugin %s with ID %s was referenced '
'in the index but not found in the current '
'implementation. Ignored.' % (plugin, id))
def test_construction():
"""If a plugin is a plain module, make sure it is automatically promoted to
a Plugin via recognition of symbol naming conventions."""
plugin = all_plugins()['urllink']
mocked_tree = None # This will probably have to improve at some point.
mocked_vcs = None
ok_(
isinstance(
plugin.tree_to_index('urllink', mocked_tree,
mocked_vcs).file_to_index('/foo/bar', ''),
urllink.FileToIndex))
def plugin_list(value):
"""Turn a space-delimited series of plugin names into a ListAndAll of
Plugins.
"""
if not isinstance(value, basestring):
raise SchemaError('"%s" is neither * nor a whitespace-delimited list '
'of plugin names.' % (value,))
plugins = all_plugins()
names = value.strip().split()
is_all = names == ['*']
if is_all:
names = plugins.keys()
try:
ret = ListAndAll([plugins[name] for name in names])
ret.is_all = is_all
return ret
except KeyError:
raise SchemaError('Never heard of plugin "%s". I\'ve heard of '
'these: %s.' % (name, ', '.join(plugins.keys())))
def test_registration():
"""Make sure plugins registered via entry points are detected."""
ok_('urllink' in all_plugins().keys())
def test_registration():
"""Make sure plugins registered via entry points are detected."""
ok_('urllink' in all_plugins().keys())
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('generated_date',
default=datetime.utcnow()
.strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers', default=if_raises(NotImplementedError,
cpu_count,
1)):
And(Use(int),
lambda v: v >= 0,
error='"workers" must be a non-negative integer.'),
Optional('skip_stages', default=[]): WhitespaceList,
Optional('www_root', default=''): Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''): basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_indexing_timeout" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(input.splitlines() if isinstance(input,
basestring)
else input,
list_values=False)
if not relative_to:
relative_to = getcwd()
with cd(relative_to):
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins().values()
if p not in self.disabled_plugins]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(section,
config[section],
config_obj[section].sections,
self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config):
"""Fix up settings that depend on the [DXR] section or have
inter-setting dependencies. (schema can't do multi-setting validation
yet, and configobj can't do cross-section interpolation.)
Add a ``config`` attr to trees as a shortcut back to the [DXR] section
and a ``name`` attr to save cumbersome tuple unpacks in callers.
"""
self.config = config
self.name = name
schema = Schema({
Optional('build_command', default='make -j {workers}'): basestring,
Optional('clean_command', default='make clean'): basestring,
Optional('description', default=''): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('es_index', default=config.es_index): basestring,
Optional('es_shards', default=5):
Use(int, error='"es_shards" must be an integer.'),
Optional('ignore_patterns',
default=['.hg', '.git', 'CVS', '.svn', '.bzr',
'.deps', '.libs', '.DS_Store', '.nfs*', '*~',
'._*']): WhitespaceList,
Optional('object_folder', default=None): AbsPath,
'source_folder': AbsPath,
Optional('source_encoding', default='utf-8'): basestring,
Optional('temp_folder', default=None): AbsPath,
Optional('p4web_url', default='http://p4web/'): basestring,
Optional(basestring): dict})
tree = schema.validate(unvalidated_tree)
if tree['temp_folder'] is None:
tree['temp_folder'] = config.temp_folder
if tree['object_folder'] is None:
tree['object_folder'] = tree['source_folder']
# Convert enabled_plugins to a list of plugins:
if tree['disabled_plugins'].is_all:
# * doesn't really mean "all" in a tree. It means "everything the
# [DXR] section enabled".
tree['disabled_plugins'] = config.enabled_plugins
else:
# Add anything globally disabled to our local disabled list:
tree['disabled_plugins'].extend(p for p in config.disabled_plugins
if p not in
tree['disabled_plugins'])
if tree['enabled_plugins'].is_all:
tree['enabled_plugins'] = [p for p in config.enabled_plugins
if p not in tree['disabled_plugins']]
tree['enabled_plugins'].insert(0, all_plugins()['core'])
# Split ignores into paths and filenames:
tree['ignore_paths'] = [i for i in tree['ignore_patterns']
if i.startswith('/')]
tree['ignore_filenames'] = [i for i in tree['ignore_patterns']
if not i.startswith('/')]
# Delete misleading, useless, or raw values people shouldn't use:
del tree['ignore_patterns']
del tree['disabled_plugins']
# Validate plugin config:
enableds_with_all_optional_config = set(
p for p in tree['enabled_plugins']
if all(isinstance(k, Optional) for k in p.config_schema.iterkeys()))
plugin_schema = Schema(merge(
dict((Optional(name) if plugin in enableds_with_all_optional_config
or plugin not in tree['enabled_plugins']
else name,
plugin.config_schema)
for name, plugin in all_plugins().iteritems()
if name != 'core'),
# And whatever isn't a plugin section, that we don't care about:
{object: object}))
# Insert empty missing sections for enabled plugins with entirely
# optional config so their defaults get filled in. (Don't insert them
# if the plugin has any required options; then we wouldn't produce the
# proper error message about the section being absent.)
for plugin in enableds_with_all_optional_config:
tree.setdefault(plugin.name, {})
tree = plugin_schema.validate(tree)
super(TreeConfig, self).__init__(tree)
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None):
"""Return a rendered page displaying a source file.
:arg string tree: name of tree on which file is found
:arg string path: relative path from tree root of file
:arg list line_docs: LINE documents as defined in the mapping of core.py,
where the `content` field is dereferenced
:arg file_doc: the FILE document as defined in core.py
:arg config: TreeConfig object of this tree
:arg date: a formatted string representing the generated date, default to now
:arg string contents: the contents of the source file, defaults to joining
the `content` field of all line_docs
"""
def sidebar_links(sections):
"""Return data structure to build nav sidebar from. ::
[('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])]
"""
# Sort by order, resolving ties by section name:
return sorted(sections, key=lambda section: (section['order'],
section['heading']))
if not date:
# Then assume that the file is generated now. Remark: we can't use this
# as the default param because that is only evaluated once, so the same
# time would always be used.
date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
common = _build_common_file_template(tree, path, date, config)
links = file_doc.get('links', [])
if is_image(path):
return render_template(
'image_file.html',
**common)
else: # We don't allow browsing binary files, so this must be a text file.
# We concretize the lines into a list because we iterate over it multiple times
lines = [doc['content'] for doc in line_docs]
if not contents:
# If contents are not provided, we can reconstruct them by
# stitching the lines together.
contents = ''.join(lines)
offsets = cumulative_sum(imap(len, lines))
# Construct skimmer objects for all enabled plugins that define a
# file_to_skim class.
skimmers = [plugin.file_to_skim(path,
contents,
name,
config.trees[tree],
file_doc,
line_docs)
for name, plugin in all_plugins().iteritems()
if plugin in config.trees[tree].enabled_plugins
and plugin.file_to_skim]
skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs))
index_refs = imap(Ref.es_to_triple,
chain.from_iterable(doc.get('refs', [])
for doc in line_docs))
index_regions = imap(Region.es_to_triple,
chain.from_iterable(doc.get('regions', [])
for doc in line_docs))
tags = finished_tags(lines,
chain(chain.from_iterable(refses), index_refs),
chain(chain.from_iterable(regionses), index_regions))
return render_template(
'text_file.html',
**merge(common, {
# Someday, it would be great to stream this and not concretize
# the whole thing in RAM. The template will have to quit
# looping through the whole thing 3 times.
'lines': [(html_line(doc['content'], tags_in_line, offset),
doc.get('annotations', []) + skim_annotations)
for doc, tags_in_line, offset, skim_annotations
in izip(line_docs, tags_per_line(tags), offsets, annotationses)],
'is_text': True,
'sections': sidebar_links(links + skim_links)}))