Exemplo n.º 1
0
def test_construction():
    """If a plugin is a plain module, make sure it is automatically promoted to
    a Plugin via recognition of symbol naming conventions."""
    plugin = all_plugins()['urllink']
    mocked_tree = None  # This will probably have to improve at some point.
    mocked_vcs = None
    ok_(isinstance(plugin.tree_to_index('urllink', mocked_tree, mocked_vcs).file_to_index('/foo/bar', ''),
                   urllink.FileToIndex))
Exemplo n.º 2
0
 def ref_class(plugin, id):
     """Return the subclass of Ref identified by a combination of
     plugin and class ID."""
     plugins = all_plugins()
     try:
         return plugins[plugin].refs[id]
     except KeyError:
         warn('Ref subclass from plugin %s with ID %s was referenced '
              'in the index but not found in the current '
              'implementation. Ignored.' % (plugin, id))
Exemplo n.º 3
0
 def ref_class(plugin, id):
     """Return the subclass of Ref identified by a combination of
     plugin and class ID."""
     plugins = all_plugins()
     try:
         return plugins[plugin].refs[id]
     except KeyError:
         warn('Ref subclass from plugin %s with ID %s was referenced '
              'in the index but not found in the current '
              'implementation. Ignored.' % (plugin, id))
Exemplo n.º 4
0
def test_construction():
    """If a plugin is a plain module, make sure it is automatically promoted to
    a Plugin via recognition of symbol naming conventions."""
    plugin = all_plugins()['urllink']
    mocked_tree = None  # This will probably have to improve at some point.
    mocked_vcs = None
    ok_(
        isinstance(
            plugin.tree_to_index('urllink', mocked_tree,
                                 mocked_vcs).file_to_index('/foo/bar', ''),
            urllink.FileToIndex))
Exemplo n.º 5
0
Arquivo: config.py Projeto: nbstar/dxr
def plugin_list(value):
    """Turn a space-delimited series of plugin names into a ListAndAll of
    Plugins.

    """
    if not isinstance(value, basestring):
        raise SchemaError('"%s" is neither * nor a whitespace-delimited list '
                          'of plugin names.' % (value,))

    plugins = all_plugins()
    names = value.strip().split()
    is_all = names == ['*']
    if is_all:
        names = plugins.keys()
    try:
        ret = ListAndAll([plugins[name] for name in names])
        ret.is_all = is_all
        return ret
    except KeyError:
        raise SchemaError('Never heard of plugin "%s". I\'ve heard of '
                          'these: %s.' % (name, ', '.join(plugins.keys())))
Exemplo n.º 6
0
def test_registration():
    """Make sure plugins registered via entry points are detected."""
    ok_('urllink' in all_plugins().keys())
Exemplo n.º 7
0
def test_registration():
    """Make sure plugins registered via entry points are detected."""
    ok_('urllink' in all_plugins().keys())
Exemplo n.º 8
0
Arquivo: config.py Projeto: nbstar/dxr
    def __init__(self, input, relative_to=None):
        """Pull in and validate a config file.

        :arg input: A string or dict from which to populate the config
        :arg relative_to: The dir relative to which to interpret relative paths

        Raise ConfigError if the configuration is invalid.

        """
        schema = Schema({
            'DXR': {
                Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
                    AbsPath,
                Optional('default_tree', default=None): basestring,
                Optional('disabled_plugins', default=plugin_list('')): Plugins,
                Optional('enabled_plugins', default=plugin_list('*')): Plugins,
                Optional('generated_date',
                         default=datetime.utcnow()
                                         .strftime("%a, %d %b %Y %H:%M:%S +0000")):
                    basestring,
                Optional('log_folder', default=abspath('dxr-logs-{tree}')):
                    AbsPath,
                Optional('workers', default=if_raises(NotImplementedError,
                                                      cpu_count,
                                                      1)):
                    And(Use(int),
                        lambda v: v >= 0,
                        error='"workers" must be a non-negative integer.'),
                Optional('skip_stages', default=[]): WhitespaceList,
                Optional('www_root', default=''): Use(lambda v: v.rstrip('/')),
                Optional('google_analytics_key', default=''): basestring,
                Optional('es_hosts', default='http://127.0.0.1:9200/'):
                    WhitespaceList,
                # A semi-random name, having the tree name and format version in it.
                Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
                    basestring,
                Optional('es_alias', default='dxr_{format}_{tree}'):
                    basestring,
                Optional('es_catalog_index', default='dxr_catalog'):
                    basestring,
                Optional('es_catalog_replicas', default=1):
                    Use(int, error='"es_catalog_replicas" must be an integer.'),
                Optional('max_thumbnail_size', default=20000):
                    And(Use(int),
                        lambda v: v >= 0,
                        error='"max_thumbnail_size" must be a non-negative '
                              'integer.'),
                Optional('es_indexing_timeout', default=60):
                    And(Use(int),
                        lambda v: v >= 0,
                        error='"es_indexing_timeout" must be a non-negative '
                              'integer.'),
                Optional('es_refresh_interval', default=60):
                    Use(int, error='"es_indexing_timeout" must be an integer.')
            },
            basestring: dict
        })

        # Parse the ini into nested dicts:
        config_obj = ConfigObj(input.splitlines() if isinstance(input,
                                                                basestring)
                               else input,
                               list_values=False)

        if not relative_to:
            relative_to = getcwd()
        with cd(relative_to):
            try:
                config = schema.validate(config_obj.dict())
            except SchemaError as exc:
                raise ConfigError(exc.code, ['DXR'])

            self._section = config['DXR']

            # Normalize enabled_plugins:
            if self.enabled_plugins.is_all:
                # Then explicitly enable anything that isn't explicitly
                # disabled:
                self._section['enabled_plugins'] = [
                        p for p in all_plugins().values()
                        if p not in self.disabled_plugins]

            # Now that enabled_plugins and the other keys that TreeConfig
            # depends on are filled out, make some TreeConfigs:
            self.trees = OrderedDict()  # name -> TreeConfig
            for section in config_obj.sections:
                if section != 'DXR':
                    try:
                        self.trees[section] = TreeConfig(section,
                                                         config[section],
                                                         config_obj[section].sections,
                                                         self)
                    except SchemaError as exc:
                        raise ConfigError(exc.code, [section])

        # Make sure default_tree is defined:
        if not self.default_tree:
            self._section['default_tree'] = first(self.trees.iterkeys())

        # These aren't intended for actual use; they're just to influence
        # enabled_plugins of trees, and now we're done with them:
        del self._section['enabled_plugins']
        del self._section['disabled_plugins']
Exemplo n.º 9
0
Arquivo: config.py Projeto: nbstar/dxr
    def __init__(self, name, unvalidated_tree, sections, config):
        """Fix up settings that depend on the [DXR] section or have
        inter-setting dependencies. (schema can't do multi-setting validation
        yet, and configobj can't do cross-section interpolation.)

        Add a ``config`` attr to trees as a shortcut back to the [DXR] section
        and a ``name`` attr to save cumbersome tuple unpacks in callers.

        """
        self.config = config
        self.name = name

        schema = Schema({
            Optional('build_command', default='make -j {workers}'): basestring,
            Optional('clean_command', default='make clean'): basestring,
            Optional('description', default=''): basestring,
            Optional('disabled_plugins', default=plugin_list('')): Plugins,
            Optional('enabled_plugins', default=plugin_list('*')): Plugins,
            Optional('es_index', default=config.es_index): basestring,
            Optional('es_shards', default=5):
                Use(int, error='"es_shards" must be an integer.'),
            Optional('ignore_patterns',
                     default=['.hg', '.git', 'CVS', '.svn', '.bzr',
                              '.deps', '.libs', '.DS_Store', '.nfs*', '*~',
                              '._*']): WhitespaceList,
            Optional('object_folder', default=None): AbsPath,
            'source_folder': AbsPath,
            Optional('source_encoding', default='utf-8'): basestring,
            Optional('temp_folder', default=None): AbsPath,
            Optional('p4web_url', default='http://p4web/'): basestring,
            Optional(basestring): dict})
        tree = schema.validate(unvalidated_tree)

        if tree['temp_folder'] is None:
            tree['temp_folder'] = config.temp_folder
        if tree['object_folder'] is None:
            tree['object_folder'] = tree['source_folder']

        # Convert enabled_plugins to a list of plugins:
        if tree['disabled_plugins'].is_all:
            # * doesn't really mean "all" in a tree. It means "everything the
            # [DXR] section enabled".
            tree['disabled_plugins'] = config.enabled_plugins
        else:
            # Add anything globally disabled to our local disabled list:
            tree['disabled_plugins'].extend(p for p in config.disabled_plugins
                                            if p not in
                                            tree['disabled_plugins'])

        if tree['enabled_plugins'].is_all:
            tree['enabled_plugins'] = [p for p in config.enabled_plugins
                                       if p not in tree['disabled_plugins']]
        tree['enabled_plugins'].insert(0, all_plugins()['core'])

        # Split ignores into paths and filenames:
        tree['ignore_paths'] = [i for i in tree['ignore_patterns']
                                if i.startswith('/')]
        tree['ignore_filenames'] = [i for i in tree['ignore_patterns']
                                    if not i.startswith('/')]

        # Delete misleading, useless, or raw values people shouldn't use:
        del tree['ignore_patterns']
        del tree['disabled_plugins']

        # Validate plugin config:
        enableds_with_all_optional_config = set(
            p for p in tree['enabled_plugins']
            if all(isinstance(k, Optional) for k in p.config_schema.iterkeys()))
        plugin_schema = Schema(merge(
            dict((Optional(name) if plugin in enableds_with_all_optional_config
                                 or plugin not in tree['enabled_plugins']
                  else name,
                  plugin.config_schema)
                 for name, plugin in all_plugins().iteritems()
                 if name != 'core'),
            # And whatever isn't a plugin section, that we don't care about:
            {object: object}))
        # Insert empty missing sections for enabled plugins with entirely
        # optional config so their defaults get filled in. (Don't insert them
        # if the plugin has any required options; then we wouldn't produce the
        # proper error message about the section being absent.)
        for plugin in enableds_with_all_optional_config:
            tree.setdefault(plugin.name, {})
        tree = plugin_schema.validate(tree)

        super(TreeConfig, self).__init__(tree)
Exemplo n.º 10
0
Arquivo: app.py Projeto: nbstar/dxr
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None):
    """Return a rendered page displaying a source file.

    :arg string tree: name of tree on which file is found
    :arg string path: relative path from tree root of file
    :arg list line_docs: LINE documents as defined in the mapping of core.py,
        where the `content` field is dereferenced
    :arg file_doc: the FILE document as defined in core.py
    :arg config: TreeConfig object of this tree
    :arg date: a formatted string representing the generated date, default to now
    :arg string contents: the contents of the source file, defaults to joining
        the `content` field of all line_docs
    """
    def sidebar_links(sections):
        """Return data structure to build nav sidebar from. ::

            [('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])]

        """
        # Sort by order, resolving ties by section name:
        return sorted(sections, key=lambda section: (section['order'],
                                                     section['heading']))

    if not date:
        # Then assume that the file is generated now. Remark: we can't use this
        # as the default param because that is only evaluated once, so the same
        # time would always be used.
        date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")

    common = _build_common_file_template(tree, path, date, config)
    links = file_doc.get('links', [])
    if is_image(path):
        return render_template(
            'image_file.html',
            **common)
    else:  # We don't allow browsing binary files, so this must be a text file.
        # We concretize the lines into a list because we iterate over it multiple times
        lines = [doc['content'] for doc in line_docs]
        if not contents:
            # If contents are not provided, we can reconstruct them by
            # stitching the lines together.
            contents = ''.join(lines)
        offsets = cumulative_sum(imap(len, lines))
        # Construct skimmer objects for all enabled plugins that define a
        # file_to_skim class.
        skimmers = [plugin.file_to_skim(path,
                                        contents,
                                        name,
                                        config.trees[tree],
                                        file_doc,
                                        line_docs)
                    for name, plugin in all_plugins().iteritems()
                    if plugin in config.trees[tree].enabled_plugins
                    and plugin.file_to_skim]
        skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs))
        index_refs = imap(Ref.es_to_triple,
                          chain.from_iterable(doc.get('refs', [])
                                              for doc in line_docs))
        index_regions = imap(Region.es_to_triple,
                             chain.from_iterable(doc.get('regions', [])
                                                 for doc in line_docs))
        tags = finished_tags(lines,
                             chain(chain.from_iterable(refses), index_refs),
                             chain(chain.from_iterable(regionses), index_regions))
        return render_template(
            'text_file.html',
            **merge(common, {
                # Someday, it would be great to stream this and not concretize
                # the whole thing in RAM. The template will have to quit
                # looping through the whole thing 3 times.
                'lines': [(html_line(doc['content'], tags_in_line, offset),
                           doc.get('annotations', []) + skim_annotations)
                          for doc, tags_in_line, offset, skim_annotations
                              in izip(line_docs, tags_per_line(tags), offsets, annotationses)],
                'is_text': True,
                'sections': sidebar_links(links + skim_links)}))