def test_construction(): """If a plugin is a plain module, make sure it is automatically promoted to a Plugin via recognition of symbol naming conventions.""" plugin = all_plugins()['urllink'] mocked_tree = None # This will probably have to improve at some point. mocked_vcs = None ok_(isinstance(plugin.tree_to_index('urllink', mocked_tree, mocked_vcs).file_to_index('/foo/bar', ''), urllink.FileToIndex))
def ref_class(plugin, id): """Return the subclass of Ref identified by a combination of plugin and class ID.""" plugins = all_plugins() try: return plugins[plugin].refs[id] except KeyError: warn('Ref subclass from plugin %s with ID %s was referenced ' 'in the index but not found in the current ' 'implementation. Ignored.' % (plugin, id))
def test_construction(): """If a plugin is a plain module, make sure it is automatically promoted to a Plugin via recognition of symbol naming conventions.""" plugin = all_plugins()['urllink'] mocked_tree = None # This will probably have to improve at some point. mocked_vcs = None ok_( isinstance( plugin.tree_to_index('urllink', mocked_tree, mocked_vcs).file_to_index('/foo/bar', ''), urllink.FileToIndex))
def plugin_list(value): """Turn a space-delimited series of plugin names into a ListAndAll of Plugins. """ if not isinstance(value, basestring): raise SchemaError('"%s" is neither * nor a whitespace-delimited list ' 'of plugin names.' % (value,)) plugins = all_plugins() names = value.strip().split() is_all = names == ['*'] if is_all: names = plugins.keys() try: ret = ListAndAll([plugins[name] for name in names]) ret.is_all = is_all return ret except KeyError: raise SchemaError('Never heard of plugin "%s". I\'ve heard of ' 'these: %s.' % (name, ', '.join(plugins.keys())))
def test_registration(): """Make sure plugins registered via entry points are detected.""" ok_('urllink' in all_plugins().keys())
def __init__(self, input, relative_to=None): """Pull in and validate a config file. :arg input: A string or dict from which to populate the config :arg relative_to: The dir relative to which to interpret relative paths Raise ConfigError if the configuration is invalid. """ schema = Schema({ 'DXR': { Optional('temp_folder', default=abspath('dxr-temp-{tree}')): AbsPath, Optional('default_tree', default=None): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('generated_date', default=datetime.utcnow() .strftime("%a, %d %b %Y %H:%M:%S +0000")): basestring, Optional('log_folder', default=abspath('dxr-logs-{tree}')): AbsPath, Optional('workers', default=if_raises(NotImplementedError, cpu_count, 1)): And(Use(int), lambda v: v >= 0, error='"workers" must be a non-negative integer.'), Optional('skip_stages', default=[]): WhitespaceList, Optional('www_root', default=''): Use(lambda v: v.rstrip('/')), Optional('google_analytics_key', default=''): basestring, Optional('es_hosts', default='http://127.0.0.1:9200/'): WhitespaceList, # A semi-random name, having the tree name and format version in it. Optional('es_index', default='dxr_{format}_{tree}_{unique}'): basestring, Optional('es_alias', default='dxr_{format}_{tree}'): basestring, Optional('es_catalog_index', default='dxr_catalog'): basestring, Optional('es_catalog_replicas', default=1): Use(int, error='"es_catalog_replicas" must be an integer.'), Optional('max_thumbnail_size', default=20000): And(Use(int), lambda v: v >= 0, error='"max_thumbnail_size" must be a non-negative ' 'integer.'), Optional('es_indexing_timeout', default=60): And(Use(int), lambda v: v >= 0, error='"es_indexing_timeout" must be a non-negative ' 'integer.'), Optional('es_refresh_interval', default=60): Use(int, error='"es_indexing_timeout" must be an integer.') }, basestring: dict }) # Parse the ini into nested dicts: config_obj = ConfigObj(input.splitlines() if isinstance(input, basestring) else input, list_values=False) if not relative_to: relative_to = getcwd() with cd(relative_to): try: config = schema.validate(config_obj.dict()) except SchemaError as exc: raise ConfigError(exc.code, ['DXR']) self._section = config['DXR'] # Normalize enabled_plugins: if self.enabled_plugins.is_all: # Then explicitly enable anything that isn't explicitly # disabled: self._section['enabled_plugins'] = [ p for p in all_plugins().values() if p not in self.disabled_plugins] # Now that enabled_plugins and the other keys that TreeConfig # depends on are filled out, make some TreeConfigs: self.trees = OrderedDict() # name -> TreeConfig for section in config_obj.sections: if section != 'DXR': try: self.trees[section] = TreeConfig(section, config[section], config_obj[section].sections, self) except SchemaError as exc: raise ConfigError(exc.code, [section]) # Make sure default_tree is defined: if not self.default_tree: self._section['default_tree'] = first(self.trees.iterkeys()) # These aren't intended for actual use; they're just to influence # enabled_plugins of trees, and now we're done with them: del self._section['enabled_plugins'] del self._section['disabled_plugins']
def __init__(self, name, unvalidated_tree, sections, config): """Fix up settings that depend on the [DXR] section or have inter-setting dependencies. (schema can't do multi-setting validation yet, and configobj can't do cross-section interpolation.) Add a ``config`` attr to trees as a shortcut back to the [DXR] section and a ``name`` attr to save cumbersome tuple unpacks in callers. """ self.config = config self.name = name schema = Schema({ Optional('build_command', default='make -j {workers}'): basestring, Optional('clean_command', default='make clean'): basestring, Optional('description', default=''): basestring, Optional('disabled_plugins', default=plugin_list('')): Plugins, Optional('enabled_plugins', default=plugin_list('*')): Plugins, Optional('es_index', default=config.es_index): basestring, Optional('es_shards', default=5): Use(int, error='"es_shards" must be an integer.'), Optional('ignore_patterns', default=['.hg', '.git', 'CVS', '.svn', '.bzr', '.deps', '.libs', '.DS_Store', '.nfs*', '*~', '._*']): WhitespaceList, Optional('object_folder', default=None): AbsPath, 'source_folder': AbsPath, Optional('source_encoding', default='utf-8'): basestring, Optional('temp_folder', default=None): AbsPath, Optional('p4web_url', default='http://p4web/'): basestring, Optional(basestring): dict}) tree = schema.validate(unvalidated_tree) if tree['temp_folder'] is None: tree['temp_folder'] = config.temp_folder if tree['object_folder'] is None: tree['object_folder'] = tree['source_folder'] # Convert enabled_plugins to a list of plugins: if tree['disabled_plugins'].is_all: # * doesn't really mean "all" in a tree. It means "everything the # [DXR] section enabled". tree['disabled_plugins'] = config.enabled_plugins else: # Add anything globally disabled to our local disabled list: tree['disabled_plugins'].extend(p for p in config.disabled_plugins if p not in tree['disabled_plugins']) if tree['enabled_plugins'].is_all: tree['enabled_plugins'] = [p for p in config.enabled_plugins if p not in tree['disabled_plugins']] tree['enabled_plugins'].insert(0, all_plugins()['core']) # Split ignores into paths and filenames: tree['ignore_paths'] = [i for i in tree['ignore_patterns'] if i.startswith('/')] tree['ignore_filenames'] = [i for i in tree['ignore_patterns'] if not i.startswith('/')] # Delete misleading, useless, or raw values people shouldn't use: del tree['ignore_patterns'] del tree['disabled_plugins'] # Validate plugin config: enableds_with_all_optional_config = set( p for p in tree['enabled_plugins'] if all(isinstance(k, Optional) for k in p.config_schema.iterkeys())) plugin_schema = Schema(merge( dict((Optional(name) if plugin in enableds_with_all_optional_config or plugin not in tree['enabled_plugins'] else name, plugin.config_schema) for name, plugin in all_plugins().iteritems() if name != 'core'), # And whatever isn't a plugin section, that we don't care about: {object: object})) # Insert empty missing sections for enabled plugins with entirely # optional config so their defaults get filled in. (Don't insert them # if the plugin has any required options; then we wouldn't produce the # proper error message about the section being absent.) for plugin in enableds_with_all_optional_config: tree.setdefault(plugin.name, {}) tree = plugin_schema.validate(tree) super(TreeConfig, self).__init__(tree)
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None): """Return a rendered page displaying a source file. :arg string tree: name of tree on which file is found :arg string path: relative path from tree root of file :arg list line_docs: LINE documents as defined in the mapping of core.py, where the `content` field is dereferenced :arg file_doc: the FILE document as defined in core.py :arg config: TreeConfig object of this tree :arg date: a formatted string representing the generated date, default to now :arg string contents: the contents of the source file, defaults to joining the `content` field of all line_docs """ def sidebar_links(sections): """Return data structure to build nav sidebar from. :: [('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])] """ # Sort by order, resolving ties by section name: return sorted(sections, key=lambda section: (section['order'], section['heading'])) if not date: # Then assume that the file is generated now. Remark: we can't use this # as the default param because that is only evaluated once, so the same # time would always be used. date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") common = _build_common_file_template(tree, path, date, config) links = file_doc.get('links', []) if is_image(path): return render_template( 'image_file.html', **common) else: # We don't allow browsing binary files, so this must be a text file. # We concretize the lines into a list because we iterate over it multiple times lines = [doc['content'] for doc in line_docs] if not contents: # If contents are not provided, we can reconstruct them by # stitching the lines together. contents = ''.join(lines) offsets = cumulative_sum(imap(len, lines)) # Construct skimmer objects for all enabled plugins that define a # file_to_skim class. skimmers = [plugin.file_to_skim(path, contents, name, config.trees[tree], file_doc, line_docs) for name, plugin in all_plugins().iteritems() if plugin in config.trees[tree].enabled_plugins and plugin.file_to_skim] skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs)) index_refs = imap(Ref.es_to_triple, chain.from_iterable(doc.get('refs', []) for doc in line_docs)) index_regions = imap(Region.es_to_triple, chain.from_iterable(doc.get('regions', []) for doc in line_docs)) tags = finished_tags(lines, chain(chain.from_iterable(refses), index_refs), chain(chain.from_iterable(regionses), index_regions)) return render_template( 'text_file.html', **merge(common, { # Someday, it would be great to stream this and not concretize # the whole thing in RAM. The template will have to quit # looping through the whole thing 3 times. 'lines': [(html_line(doc['content'], tags_in_line, offset), doc.get('annotations', []) + skim_annotations) for doc, tags_in_line, offset, skim_annotations in izip(line_docs, tags_per_line(tags), offsets, annotationses)], 'is_text': True, 'sections': sidebar_links(links + skim_links)}))