Пример #1
0
    def get_dependencies(self):
        """
        Retrieve the set of all dependencies for a given configuration.

        Returns:
            utils.utils.OrderedSet: The set of all dependencies for the
                tracked configuration.
        """
        all_deps = OrderedSet()
        for key, _ in self.__config.items():
            if key in self.__cli:
                continue

            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1])

        for key, _ in self.__cli.items():
            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1])

        if self.__conf_file is not None:
            all_deps.add(self.__conf_file)

        all_deps.add(self.get_path("sitemap", rel_to_cwd=True))

        cwd = os.getcwd()
        return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
Пример #2
0
    def get_stale_files(self, all_files, fileset_name):
        """
        Banana banana
        """
        stale = OrderedSet()

        previous_mtimes = self.mtimes[fileset_name]
        new_mtimes = defaultdict()

        for filename in all_files:
            mtime = get_mtime(filename)
            prev_mtime = previous_mtimes.pop(filename, None)
            new_mtimes[filename] = mtime
            if mtime == prev_mtime:
                continue

            stale.add(filename)

        self.mtimes[fileset_name] = new_mtimes

        unlisted = set(previous_mtimes.keys())

        ChangeTracker.all_stale_files |= stale
        ChangeTracker.all_unlisted_files |= unlisted

        return stale, unlisted
Пример #3
0
    def get_dependencies(self):
        """
        Retrieve the set of all dependencies for a given configuration.

        Returns:
            utils.utils.OrderedSet: The set of all dependencies for the
                tracked configuration.
        """
        all_deps = OrderedSet()
        for key, _ in list(self.__config.items()):
            if key in self.__cli:
                continue

            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1 - 1])

        for key, _ in list(self.__cli.items()):
            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1 - 1])

        if self.conf_file is not None:
            all_deps.add(self.conf_file)

        all_deps.add(self.get_path("sitemap", rel_to_cwd=True))

        cwd = os.getcwd()
        return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
Пример #4
0
    def get_stale_files(self, all_files, fileset_name):
        """
        Banana banana
        """
        stale = OrderedSet()

        previous_mtimes = self.mtimes[fileset_name]
        new_mtimes = defaultdict()

        for filename in all_files:
            mtime = get_mtime(filename)
            prev_mtime = previous_mtimes.pop(filename, None)
            new_mtimes[filename] = mtime
            if mtime == prev_mtime:
                continue

            stale.add(filename)

        self.mtimes[fileset_name] = new_mtimes

        unlisted = set(previous_mtimes.keys())

        ChangeTracker.all_stale_files |= stale
        ChangeTracker.all_unlisted_files |= unlisted

        return stale, unlisted
Пример #5
0
    def __resolve_patterns(self, source_patterns, from_conf):
        if source_patterns is None:
            return OrderedSet()

        all_files = OrderedSet()
        for item in source_patterns:
            item = self.__abspath(item, from_conf)
            if '*' in item:
                all_files |= glob.glob(item)
            else:
                all_files.add(item)

        return all_files
Пример #6
0
    def get_markdown_files(self, dir_):
        """
        Get all the markdown files in a folder, recursively

        Args:
            dir_: str, a toplevel folder to walk.
        """
        md_files = OrderedSet()
        for root, _, files in os.walk(dir_):
            for name in files:
                split = os.path.splitext(name)
                if len(split) == 1:
                    continue
                if split[1] in ('.markdown', '.md', '.yaml'):
                    md_files.add(os.path.join(root, name))
        return md_files
Пример #7
0
    def get_markdown_files(self, dir_):
        """
        Get all the markdown files in a folder, recursively

        Args:
            dir_: str, a toplevel folder to walk.
        """
        md_files = OrderedSet()
        for root, _, files in os.walk(dir_):
            for name in files:
                split = os.path.splitext(name)
                if len(split) == 1:
                    continue
                if split[1] in ('.markdown', '.md', '.yaml'):
                    md_files.add(os.path.join(root, name))
        return md_files
Пример #8
0
    def __resolve_patterns(self, source_patterns, from_conf):
        if source_patterns is None:
            return OrderedSet()

        all_files = OrderedSet()
        for item in source_patterns:
            item = self.__abspath(item, from_conf)

            if item in all_files:
                continue

            if not os.path.exists(item):
                all_files |= glob.glob(item)
            else:
                all_files.add(item)

        return all_files
Пример #9
0
class Page(object):
    "Banana banana"
    meta_schema = {
        Optional('title'): And(unicode, len),
        Optional('symbols'): Schema([And(unicode, len)]),
        Optional('short-description'): And(unicode, len)
    }

    resolving_symbol_signal = Signal()
    formatting_signal = Signal()

    def __init__(self, source_file, ast, meta=None, raw_contents=None):
        "Banana banana"
        assert source_file
        if os.path.isabs(source_file):
            basename = os.path.basename(source_file)
        else:
            basename = source_file.replace('/', '-')
        name = os.path.splitext(basename)[0]
        pagename = '%s.html' % name

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.typed_symbols = {}
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None

        meta = meta or {}

        try:
            self.meta = Schema(Page.meta_schema).validate(meta)
        except SchemaError as _:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
            self.meta = meta

        self.symbol_names = OrderedSet(meta.get('symbols') or [])
        self.short_description = meta.get('short-description')

        self.title = None
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, name)

    def __getstate__(self):
        return {
            'ast': None,
            'title': self.title,
            'raw_contents': self.raw_contents,
            'short_description': self.short_description,
            'extension_name': self.extension_name,
            'link': self.link,
            'meta': self.meta,
            'source_file': self.source_file,
            'comment': self.comment,
            'generated': self.generated,
            'is_stale': False,
            'formatted_contents': None,
            'detailed_description': None,
            'output_attrs': None,
            'symbols': [],
            'typed_symbols': {},
            'subpages': self.subpages,
            'symbol_names': self.symbol_names
        }

    def resolve_symbols(self, doc_database, link_resolver):
        """
        When this method is called, the page's symbol names are queried
        from `doc_database`, and added to lists of actual symbols, sorted
        by symbol class.
        """
        typed_symbols_list = namedtuple('TypedSymbolsList',
                                        ['name', 'symbols'])
        self.typed_symbols[Symbol] = typed_symbols_list('FIXME symbols', [])
        self.typed_symbols[FunctionSymbol] = typed_symbols_list(
            "Functions", [])
        self.typed_symbols[CallbackSymbol] = typed_symbols_list(
            "Callback Functions", [])
        self.typed_symbols[FunctionMacroSymbol] = typed_symbols_list(
            "Function Macros", [])
        self.typed_symbols[ConstantSymbol] = typed_symbols_list(
            "Constants", [])
        self.typed_symbols[ExportedVariableSymbol] = typed_symbols_list(
            "Exported Variables", [])
        self.typed_symbols[StructSymbol] = typed_symbols_list(
            "Data Structures", [])
        self.typed_symbols[EnumSymbol] = typed_symbols_list("Enumerations", [])
        self.typed_symbols[AliasSymbol] = typed_symbols_list("Aliases", [])
        self.typed_symbols[SignalSymbol] = typed_symbols_list("Signals", [])
        self.typed_symbols[PropertySymbol] = typed_symbols_list(
            "Properties", [])
        self.typed_symbols[VFunctionSymbol] = typed_symbols_list(
            "Virtual Methods", [])
        self.typed_symbols[ClassSymbol] = typed_symbols_list("Classes", [])
        self.typed_symbols[InterfaceSymbol] = typed_symbols_list(
            "Interfaces", [])

        all_syms = OrderedSet()
        for sym_name in self.symbol_names:
            sym = doc_database.get_symbol(sym_name)
            self.__query_extra_symbols(sym, all_syms, link_resolver,
                                       doc_database)

        for sym in all_syms:
            sym.update_children_comments()
            self.__resolve_symbol(sym, link_resolver)
            self.symbol_names.add(sym.unique_name)

        class_syms = self.typed_symbols[ClassSymbol].symbols
        interface_syms = self.typed_symbols[InterfaceSymbol].symbols
        struct_syms = self.typed_symbols[StructSymbol].symbols

        if self.title is None:
            if class_syms:
                self.title = class_syms[0].display_name
            elif interface_syms:
                self.title = interface_syms[0].display_name
            elif struct_syms:
                self.title = struct_syms[0].display_name

        if self.comment is None:
            if class_syms and class_syms[0].comment:
                self.comment = class_syms[0].comment
            elif interface_syms and interface_syms[0].comment:
                self.comment = interface_syms[0].comment
            elif struct_syms and struct_syms[0].comment:
                self.comment = struct_syms[0].comment

    # pylint: disable=no-self-use
    def __fetch_comment(self, sym, doc_database):
        old_comment = sym.comment
        new_comment = doc_database.get_comment(sym.unique_name)
        sym.comment = Comment(sym.unique_name)

        if new_comment:
            sym.comment = new_comment
        elif old_comment:
            if not old_comment.filename in (ChangeTracker.all_stale_files |
                                            ChangeTracker.all_unlisted_files):
                sym.comment = old_comment

    def __format_page_comment(self, formatter, link_resolver):
        if not self.comment:
            return

        if self.comment.short_description:
            self.short_description = formatter.format_comment(
                self.comment.short_description, link_resolver).strip()
            if self.short_description.startswith('<p>'):
                self.short_description = self.short_description[3:-4]
        if self.comment.title:
            self.title = formatter.format_comment(self.comment.title,
                                                  link_resolver).strip()
            if self.title.startswith('<p>'):
                self.title = self.title[3:-4]

        if self.title:
            self.formatted_contents += '<h1>%s</h1>' % self.title

        self.formatted_contents += formatter.format_comment(
            self.comment, link_resolver)

    def format(self, formatter, link_resolver, output):
        """
        Banana banana
        """

        if not self.title and self.source_file:
            title = os.path.splitext(self.source_file)[0]
            self.title = os.path.basename(title).replace('-', ' ')

        self.formatted_contents = u''

        if self.ast:
            out, diags = cmark.ast_to_html(self.ast, link_resolver)
            for diag in diags:
                warn(diag.code,
                     message=diag.message,
                     filename=self.source_file)

            self.formatted_contents += out
        else:
            self.__format_page_comment(formatter, link_resolver)

        self.output_attrs = defaultdict(lambda: defaultdict(dict))
        formatter.prepare_page_attributes(self)
        Page.formatting_signal(self, formatter)
        self.__format_symbols(formatter, link_resolver)
        self.detailed_description =\
            formatter.format_page(self)[0]

        if output:
            formatter.write_page(self, output)

    # pylint: disable=no-self-use
    def get_title(self):
        """
        Banana banana
        """
        return self.title or 'unnamed'

    def __discover_title(self, meta):
        if meta is not None and 'title' in meta:
            self.title = meta['title']
        elif self.ast:
            self.title = cmark.title_from_ast(self.ast)

    def __format_symbols(self, formatter, link_resolver):
        for symbol in self.symbols:
            if symbol is None:
                continue
            debug(
                'Formatting symbol %s in page %s' %
                (symbol.unique_name, self.source_file), 'formatting')
            symbol.skip = not formatter.format_symbol(symbol, link_resolver)

    def __query_extra_symbols(self, sym, all_syms, link_resolver,
                              doc_database):
        if sym:
            self.__fetch_comment(sym, doc_database)
            new_symbols = sum(Page.resolving_symbol_signal(self, sym), [])
            all_syms.add(sym)

            for symbol in new_symbols:
                self.__query_extra_symbols(symbol, all_syms, link_resolver,
                                           doc_database)

    def __resolve_symbol(self, symbol, link_resolver):
        symbol.resolve_links(link_resolver)

        symbol.link.ref = "%s#%s" % (self.link.ref, symbol.unique_name)

        for link in symbol.get_extra_links():
            link.ref = "%s#%s" % (self.link.ref, link.id_)

        tsl = self.typed_symbols[type(symbol)]
        tsl.symbols.append(symbol)
        self.symbols.append(symbol)

        debug(
            'Resolved symbol %s to page %s' %
            (symbol.display_name, self.link.ref), 'resolution')
Пример #10
0
class TestDocTree(unittest.TestCase):
    def setUp(self):
        here = os.path.dirname(__file__)
        self.__md_dir = os.path.abspath(os.path.join(
            here, 'tmp-markdown-files'))
        self.__priv_dir = os.path.abspath(os.path.join(
            here, 'tmp-private'))
        self.__src_dir = os.path.abspath(os.path.join(
            here, 'tmp-src-files'))
        self.__output_dir = os.path.abspath(os.path.join(
            here, 'tmp-output'))
        self.__remove_tmp_dirs()
        os.mkdir(self.__md_dir)
        os.mkdir(self.__priv_dir)
        os.mkdir(self.__src_dir)
        os.mkdir(self.get_generated_doc_folder())
        self.include_paths = OrderedSet([self.__md_dir])
        self.include_paths.add(self.get_generated_doc_folder())

        # Using the real doc database is too costly, tests should be lightning
        # fast (and they are)
        self.doc_database = DocDatabase()
        self.doc_database.setup(self.__priv_dir)
        self.link_resolver = LinkResolver(self.doc_database)

        self.change_tracker = ChangeTracker()

        self.sitemap_parser = SitemapParser()

        self.test_ext = TestExtension(self)
        self.core_ext = CoreExtension(self)

    def tearDown(self):
        self.__remove_tmp_dirs()
        del self.test_ext
        del self.core_ext

    def get_generated_doc_folder(self):
        return os.path.join(self.__priv_dir, 'generated')

    def get_base_doc_folder(self):
        return self.__md_dir

    def get_private_folder(self):
        return self.__priv_dir

    def __parse_sitemap(self, text):
        path = os.path.join(self.__md_dir, 'sitemap.txt')
        with io.open(path, 'w', encoding='utf-8') as _:
            _.write(text)
        return self.sitemap_parser.parse(path)

    def __create_md_file(self, name, contents):
        path = os.path.join(self.__md_dir, name)
        with open(path, 'w') as _:
            _.write(contents)

        # Just making sure we don't hit a race condition,
        # in real world situations it is assumed users
        # will not update source files twice in the same
        # microsecond
        touch(path)

    def __create_src_file(self, name, symbols):
        path = os.path.join(self.__md_dir, name)
        with open(path, 'w') as _:
            for symbol in symbols:
                _.write('%s\n' % symbol)

        # Just making sure we don't hit a race condition,
        # in real world situations it is assumed users
        # will not update source files twice in the same
        # microsecond
        touch(path)

        return path

    def __remove_src_file(self, name):
        path = os.path.join(self.__md_dir, name)
        os.unlink(path)

    def __remove_md_file(self, name):
        path = os.path.join(self.__md_dir, name)
        os.unlink(path)

    def __touch_src_file(self, name):
        path = os.path.join(self.__md_dir, name)
        touch(path)

    def __remove_tmp_dirs(self):
        shutil.rmtree(self.__md_dir, ignore_errors=True)
        shutil.rmtree(self.__priv_dir, ignore_errors=True)
        shutil.rmtree(self.__src_dir, ignore_errors=True)
        shutil.rmtree(self.__output_dir, ignore_errors=True)
        shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)

    def test_basic(self):
        inp = (u'index.markdown\n'
               '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'section.markdown',
            (u'# My section\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        pages = doc_tree.get_pages()

        # We do not care about ordering
        self.assertSetEqual(
            set(pages.keys()),
            set([u'index.markdown',
                 u'section.markdown']))

        index = pages.get('index.markdown')
        self.assertEqual(index.title, u'My documentation')

    def test_basic_incremental(self):
        inp = (u'index.markdown\n'
               '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'section.markdown',
            (u'# My section\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        # Building from scratch, all pages are stale
        self.assertSetEqual(
            set(doc_tree.get_stale_pages()),
            set([u'index.markdown',
                 u'section.markdown']))

        doc_tree.persist()

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        # Nothing changed, no page is stale
        self.assertSetEqual(
            set(doc_tree.get_stale_pages()),
            set({}))

        # But we still have our pages
        self.assertSetEqual(
            set(doc_tree.get_pages()),
            set([u'index.markdown',
                 u'section.markdown']))

        touch(os.path.join(self.__md_dir, u'section.markdown'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        self.assertSetEqual(
            set(doc_tree.get_stale_pages()),
            set([u'section.markdown']))

    def __assert_extension_names(self, doc_tree, name_map):
        pages = doc_tree.get_pages()
        for name, ext_name in name_map.items():
            page = pages[name]
            self.assertEqual(ext_name, page.extension_name)

    def __assert_stale(self, doc_tree, expected_stale):
        stale_pages = doc_tree.get_stale_pages()
        for pagename in expected_stale:
            self.assertIn(pagename, stale_pages)
            stale_pages.pop(pagename)
        self.assertEqual(len(stale_pages), 0)

    def __create_test_layout(self):
        inp = (u'index.markdown\n'
               '\ttest-index\n'
               '\t\ttest-section.markdown\n'
               '\t\t\tsource_a.test\n'
               '\t\tpage_x.markdown\n'
               '\t\tpage_y.markdown\n'
               '\tcore_page.markdown\n')

        sources = []

        sources.append(self.__create_src_file(
            'source_a.test',
            ['symbol_1',
             'symbol_2']))

        sources.append(self.__create_src_file(
            'source_b.test',
            ['symbol_3',
             'symbol_4']))

        self.test_ext.index = 'test-index.markdown'
        self.test_ext.sources = sources
        self.test_ext.setup()

        sitemap = self.__parse_sitemap(inp)

        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'core_page.markdown',
            (u'# My non-extension page\n'))
        self.__create_md_file(
            'test-index.markdown',
            (u'# My test index\n'))
        self.__create_md_file(
            'test-section.markdown',
            (u'# My test section\n'
             '\n'
             'Linking to [a generated page](source_a.test)\n'))
        self.__create_md_file(
            'page_x.markdown',
            (u'---\n'
             'symbols: [symbol_3]\n'
             '...\n'
             '# Page X\n'))
        self.__create_md_file(
            'page_y.markdown',
            (u'# Page Y\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        return doc_tree, sitemap

    def __update_test_layout(self, doc_tree, sitemap):
        self.test_ext.reset()
        self.test_ext.setup()

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)
        return doc_tree

    def test_extension_basic(self):
        doc_tree, _ = self.__create_test_layout()
        self.__assert_extension_names(
            doc_tree,
            {u'index.markdown': 'core',
             u'test-index': 'test-extension',
             u'test-section.markdown': 'test-extension',
             u'source_a.test': 'test-extension',
             u'source_b.test': 'test-extension',
             u'page_x.markdown': 'test-extension',
             u'page_y.markdown': 'test-extension',
             u'core_page.markdown': 'core'})

        all_pages = doc_tree.get_pages()
        self.assertEqual(len(all_pages), 8)
        self.__assert_stale(doc_tree, all_pages)
        self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
        self.assertIn('source_a.test',
                      all_pages['test-section.markdown'].subpages)

    def test_extension_override(self):
        self.__create_md_file(
            'source_a.test.markdown',
            (u'# My override\n'))
        doc_tree, _ = self.__create_test_layout()
        page = doc_tree.get_pages()['source_a.test']

        self.assertEqual(
            page.symbol_names,
            OrderedSet(['symbol_1',
                        'symbol_2']))

        self.assertEqual(
            os.path.basename(page.source_file),
            'source_a.test.markdown')

        self.assertEqual(
            cmark.ast_to_html(page.ast, None),
            u'<h1>My override</h1>\n')

    def test_parse_yaml(self):
        inp = (u'index.markdown\n')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'---\n'
             'title: A random title\n'
             'symbols: [symbol_1, symbol_2]\n'
             '...\n'
             '# My documentation\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)

        pages = doc_tree.get_pages()
        page = pages.get('index.markdown')
        self.assertEqual(
            cmark.ast_to_html(page.ast, None),
            u'<h1>My documentation</h1>\n')

        self.assertEqual(page.title, u'A random title')

        self.assertEqual(
            page.symbol_names,
            OrderedSet(['symbol_1',
                        'symbol_2']))

    def test_empty_link_resolution(self):
        inp = (u'index.markdown\n'
               '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'section.markdown',
            (u'# My section\n'
             '\n'
             '[](index.markdown)\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)
        doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
        doc_tree.format(
            self.link_resolver, self.__output_dir,
            {self.core_ext.extension_name: self.core_ext})

        pages = doc_tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents,
            u'<h1>My section</h1>\n'
            '<p><a href="index.html">My documentation</a></p>\n')

    def test_labeled_link_resolution(self):
        inp = (u'index.markdown\n'
               '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'section.markdown',
            (u'# My section\n'
             '\n'
             '[a label](index.markdown)\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)
        doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
        doc_tree.format(
            self.link_resolver, self.__output_dir,
            {self.core_ext.extension_name: self.core_ext})

        pages = doc_tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents,
            u'<h1>My section</h1>\n'
            '<p><a href="index.html">a label</a></p>\n')

    def test_anchored_link_resolution(self):
        inp = (u'index.markdown\n'
               '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'# My documentation\n'))
        self.__create_md_file(
            'section.markdown',
            (u'# My section\n'
             '\n'
             '[](index.markdown#subsection)\n'))

        doc_tree = DocTree(self.__priv_dir, self.include_paths)
        doc_tree.parse_sitemap(self.change_tracker, sitemap)
        doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
        doc_tree.format(
            self.link_resolver, self.__output_dir,
            {self.core_ext.extension_name: self.core_ext})

        pages = doc_tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents,
            u'<h1>My section</h1>\n'
            '<p><a href="index.html#subsection">My documentation</a></p>\n')

    # pylint: disable=too-many-statements
    def test_extension_incremental(self):
        doc_tree, sitemap = self.__create_test_layout()
        doc_tree.persist()

        # Here we touch source_a.test, as its symbols were
        # all contained in a generated page, only that page
        # should now be stale
        self.__touch_src_file('source_a.test')
        doc_tree = self.__update_test_layout(doc_tree, sitemap)
        self.__assert_stale(doc_tree, set(['source_a.test']))
        doc_tree.persist()

        # We now touch source_b.test, which symbols are contained
        # both in a generated page and a user-provided one.
        # We expect both pages to be stale
        self.__touch_src_file('source_b.test')
        doc_tree = self.__update_test_layout(doc_tree, sitemap)
        self.__assert_stale(doc_tree,
                            set(['source_b.test',
                                 'page_x.markdown']))
        doc_tree.persist()

        # This one is trickier: we unlist symbol_3 from
        # page_x, which means the symbol should now be
        # documented in the generated page for source_b.test.
        # We expect both pages to be stale, and make sure
        # they contain the right symbols
        self.__create_md_file(
            'page_x.markdown',
            (u'# Page X\n'))
        doc_tree = self.__update_test_layout(doc_tree, sitemap)
        self.__assert_stale(doc_tree,
                            set(['source_b.test',
                                 'page_x.markdown']))

        page_x = doc_tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet())

        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4', 'symbol_3']))

        doc_tree.persist()

        # Let's make sure the opposite use case works as well,
        # we relocate symbol_3 in page_x , both page_x and
        # the generated page for source_b.test should be stale
        # and the symbols should be back to their original
        # layout.
        self.__create_md_file(
            'page_x.markdown',
            (u'---\n'
             'symbols: [symbol_3]\n'
             '...\n'
             '# Page X\n'))

        doc_tree = self.__update_test_layout(doc_tree, sitemap)
        self.__assert_stale(doc_tree,
                            set(['source_b.test',
                                 'page_x.markdown']))

        page_x = doc_tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4']))

        doc_tree.persist()

        # We now move the definition of symbol_3 to source_a.test,
        # we thus expect the generated page for source_a.test to be
        # stale because its source changed, same for source_b.test,
        # and page_x.markdown should be stale as well because the
        # definition of symbol_3 may have changed. The final
        # symbol layout should not have changed however.
        self.__create_src_file(
            'source_a.test',
            ['symbol_1',
             'symbol_2',
             'symbol_3'])
        self.__create_src_file(
            'source_b.test',
            ['symbol_4'])
        doc_tree = self.__update_test_layout(doc_tree, sitemap)

        self.__assert_stale(doc_tree,
                            set(['source_a.test',
                                 'source_b.test',
                                 'page_x.markdown']))

        page_x = doc_tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4']))

        source_a_page = doc_tree.get_pages()['source_a.test']
        self.assertEqual(
            source_a_page.symbol_names,
            OrderedSet(['symbol_1',
                        'symbol_2']))

        doc_tree.persist()

        # And we rollback again
        self.__create_src_file(
            'source_a.test',
            ['symbol_1',
             'symbol_2'])
        self.__create_src_file(
            'source_b.test',
            ['symbol_3',
             'symbol_4'])
        doc_tree = self.__update_test_layout(doc_tree, sitemap)

        self.__assert_stale(doc_tree,
                            set(['source_a.test',
                                 'source_b.test',
                                 'page_x.markdown']))

        page_x = doc_tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4']))

        source_a_page = doc_tree.get_pages()['source_a.test']
        self.assertEqual(
            source_a_page.symbol_names,
            OrderedSet(['symbol_1',
                        'symbol_2']))

        doc_tree.persist()

        # Now we'll try removing page_x altogether
        self.__remove_md_file('page_x.markdown')
        inp = (u'index.markdown\n'
               '\ttest-index\n'
               '\t\ttest-section.markdown\n'
               '\t\t\tsource_a.test\n'
               '\t\tpage_y.markdown\n'
               '\tcore_page.markdown\n')

        new_sitemap = self.__parse_sitemap(inp)
        doc_tree = self.__update_test_layout(doc_tree, new_sitemap)
        self.__assert_stale(doc_tree,
                            set(['source_b.test']))
        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4', 'symbol_3']))
        doc_tree.persist()

        # And rollback again
        self.__create_md_file(
            'page_x.markdown',
            (u'---\n'
             'symbols: [symbol_3]\n'
             '...\n'
             '# Page X\n'))
        doc_tree = self.__update_test_layout(doc_tree, sitemap)
        self.__assert_stale(doc_tree,
                            set(['page_x.markdown',
                                 'source_b.test']))

        page_x = doc_tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = doc_tree.get_pages()['source_b.test']
        self.assertEqual(
            source_b_page.symbol_names,
            OrderedSet(['symbol_4']))

        doc_tree.persist()
Пример #11
0
class Extension(Configurable):
    """
    All extensions should inherit from this base class

    Attributes:
        extension_name: str, the unique name of this extension, should
            be overriden and namespaced appropriately.
        project: project.Project, the Project instance which documentation
            hotdoc is working on.
        formatter: formatter.Formatter, may be subclassed.
        argument_prefix (str): Short name of this extension, used as a prefix
            to add to automatically generated command-line arguments.
    """
    # pylint: disable=unused-argument
    extension_name = "base-extension"
    argument_prefix = ''
    paths_arguments = {}
    path_arguments = {}
    written_out_sitemaps = set()

    def __init__(self, app, project):
        """Constructor for `Extension`.

        This should never get called directly.

        Args:
            project: The `project.Project` instance which documentation
                is being generated.
        """
        self.project = project
        self.app = app
        self.sources = set()
        self.smart_sources = []
        self.index = None
        self.source_roots = OrderedSet()
        self._created_symbols = DefaultOrderedDict(OrderedSet)
        self.__package_root = None
        self.__toplevel_comments = OrderedSet()

        self.formatter = self._make_formatter()

    # pylint: disable=no-self-use
    def warn(self, code, message):
        """
        Shortcut function for `loggable.warn`

        Args:
            code: see `utils.loggable.warn`
            message: see `utils.loggable.warn`
        """
        warn(code, message)

    # pylint: disable=no-self-use
    def error(self, code, message):
        """
        Shortcut function for `utils.loggable.error`

        Args:
            code: see `utils.loggable.error`
            message: see `utils.loggable.error`
        """
        error(code, message)

    def debug(self, message, domain=None):
        """
        Shortcut function for `utils.loggable.debug`

        Args:
            message: see `utils.loggable.debug`
            domain: see `utils.loggable.debug`
        """
        if domain is None:
            domain = self.extension_name
        debug(message, domain)

    def info(self, message, domain=None):
        """
        Shortcut function for `utils.loggable.info`

        Args:
            message: see `utils.loggable.info`
            domain: see `utils.loggable.info`
        """
        if domain is None:
            domain = self.extension_name
        info(message, domain)

    def reset(self):
        """
        This function is only useful for testing purposes, at least
        for now.
        """
        self._created_symbols = DefaultOrderedDict(OrderedSet)
        self.__package_root = None

    def get_pagename(self, name):
        self.__find_package_root()
        # Find the longest prefix
        longest = None
        for path in OrderedSet([self.__package_root]) | self.source_roots:
            commonprefix = os.path.commonprefix([path, name])
            if commonprefix == path and (longest is None or len(path) > len(longest)):
                longest = path

        if longest is not None:
            return os.path.relpath(name, longest)

        return name

    def get_symbol_page(self, symbol_name, symbol_pages, smart_pages, section_links):
        if symbol_name in symbol_pages:
            return symbol_pages[symbol_name]

        symbol = self.app.database.get_symbol(symbol_name)
        assert symbol is not None

        if symbol.parent_name and symbol.parent_name != symbol_name:
            page = self.get_symbol_page(
                symbol.parent_name, symbol_pages, smart_pages, section_links)
        else:
            smart_key = self._get_smart_key(symbol)
            if smart_key is None:
                return None

            if smart_key in smart_pages:
                page = smart_pages[smart_key]
            else:
                pagename = self.get_pagename(smart_key)
                page = Page(smart_key, True, self.project.sanitized_name, self.extension_name,
                            output_path=os.path.dirname(pagename))
                if page.link.ref in section_links:
                    self.warn('output-page-conflict',
                              'Creating a page for symbol %s would overwrite the page '
                              'declared in a toplevel comment (%s)' % (symbol_name, page.link.ref))
                    page = None
                else:
                    smart_pages[smart_key] = page

        if page is not None:
            symbol_pages[symbol_name] = page

        return page

    def _get_toplevel_comments(self):
        return self.__toplevel_comments

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-statements
    def make_pages(self):
        # All symbol names that no longer need to be assigned to a page
        dispatched_symbol_names = set()

        # Map symbol names with pages
        # This is used for assigning symbols with a parent to the page
        # where their parent will be rendered, unless the symbol has been
        # explicitly assigned or ignored
        symbol_pages = {}

        smart_pages = OrderedDict()

        # Map pages with the sources they explicitly import
        imported_sources = {}

        # This is simply used as a conflict detection mechanism, see
        # hotdoc.core.tests.test_doc_tree.TestTree.test_section_and_path_conflict
        section_links = set()

        # These are used as a duplicate detection mechanism, map
        # relocated or ignored symbols to the source files where they were initially
        # listed
        relocated_symbols = {}
        private_symbols = {}

        # First we make one page per toplevel comment (eg. SECTION comment)
        # This is the highest priority mechanism for sorting symbols
        for comment in self._get_toplevel_comments():
            # Programming error from extension author
            assert comment.name
            symbol_names = comment.meta.pop('symbols', [])
            private_symbol_names = comment.meta.pop('private-symbols', [])
            sources = comment.meta.pop('sources', None)

            page = Page(comment.name, True, self.project.sanitized_name, self.extension_name,
                        comment=comment)

            for symbol_name in symbol_names:
                if symbol_name in relocated_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s listed in %s was already listed in %s' %
                              (symbol_name, comment.filename, relocated_symbols[symbol_name]))
                    continue
                elif symbol_name in private_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s listed in %s was marked as private in %s' %
                              (symbol_name, comment.filename, private_symbols[symbol_name]))
                    continue
                else:
                    page.symbol_names.add(symbol_name)
                    symbol_pages[symbol_name] = page
                    relocated_symbols[symbol_name] = comment.filename
                    dispatched_symbol_names.add(symbol_name)

            for symbol_name in private_symbol_names:
                if symbol_name in relocated_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s marked private in %s was already listed in %s' %
                              (symbol_name, comment.filename, relocated_symbols[symbol_name]))
                    continue
                elif symbol_name in private_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s marked as private in %s was '
                              'already marked as private in %s' %
                              (symbol_name, comment.filename, private_symbols[symbol_name]))
                    continue
                private_symbols[symbol_name] = comment.filename
                symbol_pages[symbol_name] = None
                dispatched_symbol_names.add(symbol_name)

            section_links.add(page.link.ref)
            smart_key = self._get_comment_smart_key(comment)
            if smart_key in smart_pages:
                smart_pages[comment.name] = page
            else:
                smart_pages[smart_key] = page

            if sources is not None:
                abs_sources = []
                for source in sources:
                    if os.path.isabs(source):
                        abs_sources.append(source)
                    else:
                        abs_sources.append(os.path.abspath(os.path.join(
                            os.path.dirname(comment.filename), source)))
                imported_sources[page] = abs_sources

        # Used as a duplicate detection mechanism
        relocated_sources = {}

        # We now browse all the pages with explicitly imported sources
        # Importing sources has a lower level of priority than importing
        # symbols, which is why we do that in a separate loop
        for page, sources in imported_sources.items():
            for source in sources:
                if source not in self._get_all_sources():
                    self.warn('invalid-relocated-source',
                              'Source %s does not exist but is relocated in %s' %
                              (source, page.name))
                    continue

                if source in relocated_sources:
                    self.warn('invalid-relocated-source',
                              'Source %s relocated in %s was already relocated in %s' %
                              (source, page.name, relocated_sources[source]))
                    continue

                if source in self._created_symbols:
                    symbol_names = OrderedSet(
                        self._created_symbols[source]) - dispatched_symbol_names
                    page.symbol_names |= symbol_names
                    dispatched_symbol_names |= symbol_names

                relocated_sources[source] = page.name

        # We now browse all the symbols we have created
        for _, symbol_names in self._created_symbols.items():
            for symbol_name in symbol_names:
                if symbol_name in dispatched_symbol_names:
                    continue

                page = self.get_symbol_page(
                    symbol_name, symbol_pages, smart_pages, section_links)

                # Can be None if creating a page to hold the symbol conflicts with
                # a page explicitly declared in a toplevel comment or a parent has been
                # marked as private
                if page is None:
                    continue

                page.symbol_names.add(symbol_name)
                dispatched_symbol_names.add(symbol_name)

        # Finally we make our index page
        if self.index:
            index_page = self.project.tree.parse_page(
                self.index, self.extension_name)
        else:
            index_page = Page('%s-index' % self.argument_prefix, True, self.project.sanitized_name,
                              self.extension_name)

        if not index_page.title:
            index_page.title = self._get_smart_index_title()

        smart_pages['%s-index' % self.argument_prefix] = index_page

        return smart_pages

    def setup(self):
        """
        Extension subclasses should implement this to scan whatever
        source files they have to scan, and connect to the various
        signals they have to connect to.

        Note that this will be called *after* the `tree.Tree`
        of this instance's `Extension.project` has been fully
        constructed, but before its `tree.Tree.resolve_symbols`
        method has been called.
        """
        pass

    @staticmethod
    def get_dependencies():
        """
        Override this to return the list of extensions this extension
        depends on if needed.

        Returns:
            list: A list of `ExtDependency` instances.
        """
        return []

    def parse_toplevel_config(self, config):
        """Parses and make use of the toplevel configuration."""
        self.formatter.parse_toplevel_config(config)

    def parse_config(self, config):
        """
        Override this, making sure to chain up first, if your extension adds
        its own custom command line arguments, or you want to do any further
        processing on the automatically added arguments.

        The default implementation will set attributes on the extension:
        - 'sources': a set of absolute paths to source files for this extension
        - 'index': absolute path to the index for this extension

        Additionally, it will set an attribute for each argument added with
        `Extension.add_path_argument` or `Extension.add_paths_argument`, with
        the extension's `Extension.argument_prefix` stripped, and dashes
        changed to underscores.

        Args:
            config: a `config.Config` instance
        """
        prefix = self.argument_prefix
        self.sources = config.get_sources(prefix)
        self.smart_sources = [
            self._get_smart_filename(s) for s in self.sources]
        self.index = config.get_index(prefix)
        self.source_roots = OrderedSet(
            config.get_paths('%s_source_roots' % prefix))

        for arg, dest in list(self.paths_arguments.items()):
            val = config.get_paths(arg)
            setattr(self, dest, val)

        for arg, dest in list(self.path_arguments.items()):
            val = config.get_path(arg)
            setattr(self, dest, val)

        self.formatter.parse_config(config)

    def add_attrs(self, symbol, **kwargs):
        """
        Helper for setting symbol extension attributes
        """
        for key, val in kwargs.items():
            symbol.add_extension_attribute(self.extension_name, key, val)

    def get_attr(self, symbol, attrname):
        """
        Helper for getting symbol extension attributes
        """
        return symbol.extension_attributes.get(self.extension_name, {}).get(
            attrname, None)

    @staticmethod
    def add_arguments(parser):
        """
        Subclasses may implement this method to add their own arguments to
        the hotdoc binary.

        In this function, you should add an argument group to the passed-in
        parser, corresponding to your extension.
        You can then add arguments to that argument group.

        Example::

            @staticmethod
            def add_arguments(parser):
                group = parser.add_argument_group('Chilidoc',
                    'Delicious Hotdoc extension')
                Chilidoc.add_sources_argument(group)
                group.add_argument('--chili-spicy', action='store_true',
                    help='Whether to add extra pepper')

        Args:
            parser (argparse.ArgumentParser): Main hotdoc argument parser
        """
        pass

    @classmethod
    def get_assets_licensing(cls):
        return {}

    @classmethod
    def add_index_argument(cls, group):
        """
        Subclasses may call this to add an index argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            prefix: str, arguments have to be namespaced
        """
        prefix = cls.argument_prefix

        group.add_argument(
            '--%s-index' % prefix, action="store",
            dest="%s_index" % prefix,
            help=("Name of the %s root markdown file, can be None" % (
                cls.extension_name)))

    @classmethod
    def add_sources_argument(cls, group, allow_filters=True, prefix=None, add_root_paths=False):
        """
        Subclasses may call this to add sources and source_filters arguments.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            allow_filters: bool,  Whether the extension wishes to expose a
                source_filters argument.
            prefix: str, arguments have to be namespaced.
        """
        prefix = prefix or cls.argument_prefix

        group.add_argument("--%s-sources" % prefix,
                           action="store", nargs="+",
                           dest="%s_sources" % prefix.replace('-', '_'),
                           help="%s source files to parse" % prefix)

        if allow_filters:
            group.add_argument("--%s-source-filters" % prefix,
                               action="store", nargs="+",
                               dest="%s_source_filters" % prefix.replace(
                                   '-', '_'),
                               help="%s source files to ignore" % prefix)

        if add_root_paths:
            group.add_argument("--%s-source-roots" % prefix,
                               action="store", nargs="+",
                               dest="%s_source_roots" % prefix.replace(
                                   '-', '_'),
                               help="%s source root directories allowing files "
                                    "to be referenced relatively to those" % prefix)

    @classmethod
    def add_path_argument(cls, group, argname, dest=None, help_=None):
        """
        Subclasses may call this to expose a path argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            argname: str, the name of the argument, will be namespaced.
            dest: str, similar to the `dest` argument of
                `argparse.ArgumentParser.add_argument`, will be namespaced.
            help_: str, similar to the `help` argument of
                `argparse.ArgumentParser.add_argument`.
        """
        prefixed = '%s-%s' % (cls.argument_prefix, argname)
        if dest is None:
            dest = prefixed.replace('-', '_')
            final_dest = dest[len(cls.argument_prefix) + 1:]
        else:
            final_dest = dest
            dest = '%s_%s' % (cls.argument_prefix, dest)

        group.add_argument('--%s' % prefixed, action='store',
                           dest=dest, help=help_)
        cls.path_arguments[dest] = final_dest

    @classmethod
    def add_paths_argument(cls, group, argname, dest=None, help_=None):
        """
        Subclasses may call this to expose a paths argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            argname: str, the name of the argument, will be namespaced.
            dest: str, similar to the `dest` argument of
                `argparse.ArgumentParser.add_argument`, will be namespaced.
            help_: str, similar to the `help` argument of
                `argparse.ArgumentParser.add_argument`.
        """
        prefixed = '%s-%s' % (cls.argument_prefix, argname)
        if dest is None:
            dest = prefixed.replace('-', '_')
            final_dest = dest[len(cls.argument_prefix) + 1:]
        else:
            final_dest = dest
            dest = '%s_%s' % (cls.argument_prefix, dest)

        group.add_argument('--%s' % prefixed, action='store', nargs='+',
                           dest=dest, help=help_)
        cls.paths_arguments[dest] = final_dest

    def add_comment(self, comment):
        if comment.toplevel:
            self.__toplevel_comments.add(comment)
        else:
            self.app.database.add_comment(comment)

    def create_symbol(self, *args, **kwargs):
        """
        Extensions that discover and create instances of `symbols.Symbol`
        should do this through this method, as it will keep an index
        of these which can be used when generating a "naive index".

        See `database.Database.create_symbol` for more
        information.

        Args:
            args: see `database.Database.create_symbol`
            kwargs: see `database.Database.create_symbol`

        Returns:
            symbols.Symbol: the created symbol, or `None`.
        """
        if not kwargs.get('project_name'):
            kwargs['project_name'] = self.project.project_name

        sym = self.app.database.create_symbol(*args, **kwargs)
        if sym:
            # pylint: disable=unidiomatic-typecheck
            if type(sym) != Symbol:
                self._created_symbols[sym.filename].add(sym.unique_name)

        return sym

    def rename_symbol(self, unique_name, target):
        sym = self.app.database.rename_symbol(unique_name, target)
        # pylint: disable=unidiomatic-typecheck
        if sym and type(sym) != Symbol:
            self._created_symbols[sym.filename].remove(target)
            self._created_symbols[sym.filename].add(sym.unique_name)

    def _make_formatter(self):
        return Formatter(self)

    def get_possible_path(self, name):
        self.__find_package_root()

        for path in OrderedSet([self.__package_root]) | self.source_roots:
            possible_path = os.path.join(path, name)
            if possible_path in self._get_all_sources():
                return self._get_smart_filename(possible_path)
        return None

    def __find_package_root(self):
        if self.__package_root:
            return

        commonprefix = os.path.commonprefix(
            list(self._get_all_sources()) + list(self.source_roots))
        self.__package_root = os.path.dirname(commonprefix)

    def _get_smart_index_title(self):
        return 'Reference Manual'

    def _get_all_sources(self):
        return self.sources

    def _get_smart_key(self, symbol):
        return symbol.filename

    def _get_smart_filename(self, filename):
        return filename

    def _get_comment_smart_key(self, comment):
        return comment.filename

    def format_page(self, page, link_resolver, output):
        """
        Called by `project.Project.format_page`, to leave full control
        to extensions over the formatting of the pages they are
        responsible of.

        Args:
            page: tree.Page, the page to format.
            link_resolver: links.LinkResolver, object responsible
                for resolving links potentially mentioned in `page`
            output: str, path to the output directory.
        """
        debug('Formatting page %s' % page.link.ref, 'formatting')

        if output:
            actual_output = os.path.join(output,
                                         'html')
            if not os.path.exists(actual_output):
                os.makedirs(actual_output)
        else:
            actual_output = None

        page.format(self.formatter, link_resolver, actual_output)

    def write_out_sitemap(self, opath):
        """
        Banana banana
        """
        if opath not in self.written_out_sitemaps:
            Extension.formatted_sitemap = self.formatter.format_navigation(
                self.app.project)
            if Extension.formatted_sitemap:
                escaped_sitemap = Extension.formatted_sitemap.replace(
                    '\\', '\\\\').replace('"', '\\"').replace('\n', '')
                js_wrapper = 'sitemap_downloaded_cb("%s");' % escaped_sitemap
                with open(opath, 'w') as _:
                    _.write(js_wrapper)

        self.written_out_sitemaps.add(opath)

    # pylint: disable=too-many-locals
    def write_out_page(self, output, page):
        """
        Banana banana
        """
        subpages = OrderedDict({})
        all_pages = self.project.tree.get_pages()
        subpage_names = self.get_subpages_sorted(all_pages, page)
        for pagename in subpage_names:
            proj = self.project.subprojects.get(pagename)

            if not proj:
                cpage = all_pages[pagename]
                sub_formatter = self.project.extensions[
                    cpage.extension_name].formatter
            else:
                cpage = proj.tree.root
                sub_formatter = proj.extensions[cpage.extension_name].formatter

            subpage_link, _ = cpage.link.get_link(self.app.link_resolver)
            prefix = sub_formatter.get_output_folder(cpage)
            if prefix:
                subpage_link = '%s/%s' % (prefix, subpage_link)
            subpages[subpage_link] = cpage

        html_subpages = self.formatter.format_subpages(page, subpages)

        js_dir = os.path.join(output, 'html', 'assets', 'js')
        if not os.path.exists(js_dir):
            os.makedirs(js_dir)
        sm_path = os.path.join(js_dir, 'sitemap.js')
        self.write_out_sitemap(sm_path)

        self.formatter.write_out(page, html_subpages, output)

    def get_subpages_sorted(self, pages, page):
        """Get @page subpages sorted appropriately."""

        sorted_pages = []
        to_sort = []
        for subpage in page.subpages:
            # Do not resort subprojects even if they are
            # 'generated'.
            if pages[subpage].pre_sorted:
                sorted_pages.append(subpage)
            else:
                to_sort.append(subpage)

        return sorted_pages + sorted(
            to_sort, key=lambda p: pages[p].get_title().lower())
Пример #12
0
class GstExtension(Extension):
    extension_name = 'gst-extension'
    argument_prefix = 'gst'
    __dual_links = {}  # Maps myelement:XXX to GstMyElement:XXX
    __parsed_cfiles = set()
    __caches = {}  # cachefile -> dict
    __apps_sigs = set()
    __all_plugins_symbols = set()

    def __init__(self, app, project):
        super().__init__(app, project)
        self.cache = {}
        self.c_sources = []
        self.cache_file = None
        self.plugin = None
        self.__elements = {}
        self.__raw_comment_parser = GtkDocParser(project,
                                                 section_file_matching=False)
        self.__plugins = None
        self.__toplevel_comments = OrderedSet()
        self.list_plugins_page = None
        # If we have a plugin with only one element, we render it on the plugin
        # page.
        self.unique_feature = None
        self.__on_index_symbols = []

    def _make_formatter(self):
        return GstFormatter(self)

    def create_symbol(self, *args, **kwargs):
        sym = super().create_symbol(*args, **kwargs)
        if self.unique_feature and sym:
            self.__on_index_symbols.append(sym)

        return sym

    # pylint: disable=too-many-branches
    def setup(self):
        # Make sure the cache file is save when the whole project
        # is done.
        if self.cache_file not in GstExtension.__apps_sigs and self.cache_file:
            GstExtension.__apps_sigs.add(self.cache_file)

        if not self.cache_file:
            if self.list_plugins_page:
                self.__plugins = self.create_symbol(
                    GstPluginsSymbol,
                    display_name="All " +
                    self.project.project_name.replace("-", " ").title(),
                    unique_name=self.project.project_name + "-all-gst-plugins",
                    plugins=[],
                    all_plugins=True)

                super().setup()
            return

        gather_gtk_doc_links()

        comment_parser = GtkDocParser(self.project, False)
        to_parse_sources = set(self.c_sources) - GstExtension.__parsed_cfiles

        CCommentExtractor(self,
                          comment_parser).parse_comments(to_parse_sources)
        GstExtension.__parsed_cfiles.update(self.c_sources)

        if not self.cache:
            error('setup-issue',
                  "No cache loaded or created for %s" % self.plugin)

        plugins = []
        if self.plugin:
            pname = self.plugin
            dot_idx = pname.rfind('.')
            if dot_idx > 0:
                pname = self.plugin[:dot_idx]
            if pname.startswith('libgst'):
                pname = pname[6:]
            elif pname.startswith('gst'):
                pname = pname[3:]
            try:
                plugin_node = {pname: self.cache[pname]}
            except KeyError:
                error('setup-issue', "Plugin %s not found" % pname)
        else:
            plugin_node = self.cache

        for libfile, plugin in plugin_node.items():
            plugin_sym = self.__parse_plugin(libfile, plugin)
            if not plugin_sym:
                continue

            plugins.append(plugin_sym)

        if not self.plugin:
            self.__plugins = self.create_symbol(
                GstPluginsSymbol,
                display_name=self.project.project_name.replace("-",
                                                               " ").title(),
                unique_name=self.project.project_name + "-gst-plugins",
                plugins=plugins)

        super().setup()

    def _get_comment_smart_key(self, comment):
        try:
            return comment.title.description
        except AttributeError:
            return None

    def _get_toplevel_comments(self):
        if self.unique_feature:
            return OrderedSet()
        return self.__toplevel_comments

    def get_plugin_comment(self):
        if self.plugin:
            res = self.app.database.get_comment("plugin-" + self.plugin)
            return res
        return None

    def make_pages(self):
        smart_pages = super().make_pages()

        if not self.__plugins:
            return None

        if self.list_plugins_page is None:
            index = smart_pages.get('gst-index')
            if index is None:
                return smart_pages

            index.render_subpages = False
            index.symbol_names.add(self.__plugins.unique_name)
            for sym in self.__on_index_symbols:
                index.symbol_names.add(sym.unique_name)
            if self.unique_feature:
                index.comment = self.app.database.get_comment(
                    "element-" + self.unique_feature)
            else:
                index.comment = self.get_plugin_comment()
            return smart_pages

        page = smart_pages.get(self.list_plugins_page)
        page.render_subpages = False
        page.extension_name = self.extension_name

        page.symbol_names.add(self.__plugins.unique_name)
        self.__plugins.plugins = self.__all_plugins_symbols

        return smart_pages

    def add_comment(self, comment):
        # We handle toplevel comments ourself, make sure all comments
        # end up in the database
        comment.toplevel = False

        super().add_comment(comment)

    def _get_smart_index_title(self):
        if self.plugin:
            return self.__plugins.display_name
        return 'GStreamer plugins documentation'

    @staticmethod
    def add_arguments(parser):
        group = parser.add_argument_group('Gst extension', DESCRIPTION)
        GstExtension.add_index_argument(group)
        # GstExtension.add_order_generated_subpages(group)
        GstExtension.add_sources_argument(group, prefix='gst-c')
        group.add_argument('--gst-cache-file', default=None)
        group.add_argument('--gst-list-plugins-page', default=None)
        group.add_argument('--gst-plugin-name', default=None)
        group.add_argument('--gst-plugins-path', default=None)

    def parse_config(self, config):
        self.c_sources = config.get_sources('gst_c')
        self.cache_file = config.get('gst_cache_file')
        self.plugin = config.get('gst_plugin_name')
        self.list_plugins_page = config.get('gst_list_plugins_page', None)
        info('Parsing config!')

        self.cache = {}
        if self.cache_file:
            self.cache = GstExtension.__caches.get(self.cache_file)
            if not self.cache:
                try:
                    with open(self.cache_file) as _:
                        self.cache = json.load(_)
                except FileNotFoundError:
                    pass

                if self.cache is None:
                    self.cache = {}
                GstExtension.__caches[self.cache_file] = self.cache

        super().parse_config(config)

    def _get_smart_key(self, symbol):
        if self.unique_feature:
            return None

        if isinstance(symbol, GstPluginSymbol):
            # PluginSymbol are rendered on the index page
            return None
        res = symbol.extra.get('gst-element-name')
        if res:
            res = res.replace("element-", "")

        return res

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-arguments
    def __create_signal_symbol(self,
                               obj,
                               parent_uniquename,
                               name,
                               signal,
                               element_name,
                               parent_name=None):
        atypes = signal['args']
        instance_type = obj['hierarchy'][0]
        unique_name = "%s::%s" % (parent_uniquename, name)
        aliases = self._get_aliases(["%s::%s" % (instance_type, name)])

        gi_extension = self.project.extensions.get('gi-extension')
        python_lang = gi_extension.get_language('python')
        args_type_names = []
        args_type_names = [
            (type_tokens_from_type_name('GstElement', python_lang), 'param_0')
        ]
        for i, _ in enumerate(atypes):
            args_type_names.append(
                (type_tokens_from_type_name(atypes[i], python_lang),
                 'param_%s' % (i + 1)))

        args_type_names.append(
            (type_tokens_from_type_name('gpointer', python_lang), "udata"))
        params = []

        for comment_name in [unique_name] + aliases:
            comment = self.app.database.get_comment(comment_name)
            if comment:
                for i, argname in enumerate(comment.params.keys()):
                    args_type_names[i] = (args_type_names[i][0], argname)

        for tokens, argname in args_type_names:
            params.append(ParameterSymbol(argname=argname, type_tokens=tokens))

        type_name = signal['retval']
        if type_name == 'void':
            retval = [None]
        else:
            tokens = type_tokens_from_type_name(type_name, python_lang)

            enum = signal.get('return-values')
            if enum:
                self.__create_enum_symbol(type_name,
                                          enum,
                                          obj.get('name', parent_uniquename),
                                          parent_name=parent_name)

            retval = [ReturnItemSymbol(type_tokens=tokens)]

        return self.create_symbol(SignalSymbol,
                                  parameters=params,
                                  return_value=retval,
                                  display_name=name,
                                  unique_name=unique_name,
                                  extra={'gst-element-name': element_name},
                                  aliases=aliases,
                                  parent_name=parent_name)

    def __create_signal_symbols(self,
                                obj,
                                parent_uniquename,
                                element_name,
                                parent_name=None):
        signals = obj.get('signals', {})
        if not signals:
            return

        for name, signal in signals.items():
            self.__create_signal_symbol(obj,
                                        parent_uniquename,
                                        name,
                                        signal,
                                        element_name,
                                        parent_name=parent_name)

    def __create_property_symbols(self,
                                  obj,
                                  parent_uniquename,
                                  pagename,
                                  parent_name=None):
        properties = obj.get('properties', [])
        if not properties:
            return

        gi_extension = self.project.extensions.get('gi-extension')
        python_lang = gi_extension.get_language('python')
        for name, prop in properties.items():
            unique_name = '%s:%s' % (obj.get('name', parent_uniquename), name)
            flags = [ReadableFlag()]
            if prop['writable']:
                flags += [WritableFlag()]
            if prop['construct-only']:
                flags += [ConstructOnlyFlag()]
            elif prop['construct']:
                flags += [ConstructFlag()]

            type_name = prop['type-name']

            tokens = type_tokens_from_type_name(type_name, python_lang)
            type_ = QualifiedSymbol(type_tokens=tokens)

            default = prop.get('default')
            enum = prop.get('values')
            if enum:
                type_ = self.__create_enum_symbol(prop['type-name'],
                                                  enum,
                                                  obj.get(
                                                      'name',
                                                      parent_uniquename),
                                                  parent_name=parent_name)

            if obj['hierarchy'][0] != parent_uniquename:
                aliases = self._get_aliases(
                    ['%s:%s' % (obj['hierarchy'][0], name)])
            else:
                aliases = []

            res = self.app.database.get_symbol(unique_name)
            if res is None:
                res = self.create_symbol(
                    PropertySymbol,
                    prop_type=type_,
                    display_name=name,
                    unique_name=unique_name,
                    aliases=aliases,
                    parent_name=parent_name,
                    extra={'gst-element-name': pagename},
                )
            assert res

            if not self.app.database.get_comment(unique_name):
                comment = Comment(unique_name,
                                  Comment(name=name),
                                  description=prop['blurb'])
                self.app.database.add_comment(comment)

            # FIXME This is incorrect, it's not yet format time (from gi_extension)
            extra_content = self.formatter.format_flags(flags)
            res.extension_contents['Flags'] = extra_content
            if default:
                if prop['type-name'] in ['GstCaps', 'GstStructure']:
                    default = '<pre class="language-yaml">' + \
                        '<code class="language-yaml">%s</code></pre>' % default
                res.extension_contents['Default value'] = default

    def __create_enum_symbol(self,
                             type_name,
                             enum,
                             element_name,
                             parent_name=None):
        display_name = re.sub(r"([a-z])([A-Z])", r"\g<1>-\g<2>",
                              type_name.replace('Gst', ''))
        unique_name = type_name
        if self.app.database.get_symbol(unique_name):
            unique_name = element_name + '_' + type_name
        symbol = self.app.database.get_symbol(unique_name)
        if not symbol:
            members = []
            for val in enum:
                value_unique_name = "%s::%s" % (type_name, val['name'])
                if self.app.database.get_symbol(value_unique_name):
                    value_unique_name = "%s::%s" % (element_name + '_' +
                                                    type_name, val['name'])
                member_sym = self.create_symbol(
                    GstNamedConstantValue,
                    unique_name=value_unique_name,
                    display_name=val['name'],
                    value=val['value'],
                    parent_name=parent_name,
                    val=val,
                    extra={'gst-element-name': element_name})
                if member_sym:
                    members.append(member_sym)
            symbol = self.create_symbol(
                GstNamedConstantsSymbols,
                anonymous=False,
                raw_text=None,
                display_name=display_name.capitalize(),
                unique_name=unique_name,
                parent_name=parent_name,
                members=members,
                extra={'gst-element-name': 'element-' + element_name})
        elif not isinstance(symbol, GstNamedConstantsSymbols):
            self.warn(
                'symbol-redefined',
                "EnumMemberSymbol(unique_name=%s, project=%s)"
                " has already been defined: %s" %
                (unique_name, self.project.project_name, symbol))

            return None

        if not symbol:
            return None

        symbol.values = enum
        return symbol

    def __create_object_type(self, pagename, _object):
        if not _object:
            return None

        unique_name = _object['hierarchy'][0]
        if self.app.database.get_symbol(unique_name):
            return None

        self.__create_property_symbols(_object,
                                       unique_name,
                                       pagename,
                                       parent_name=unique_name)
        self.__create_signal_symbols(_object,
                                     unique_name,
                                     pagename,
                                     parent_name=unique_name)

        return self.create_symbol(ClassSymbol,
                                  hierarchy=create_hierarchy(_object),
                                  display_name=unique_name,
                                  unique_name=unique_name,
                                  parent_name=unique_name,
                                  extra={'gst-element-name': pagename})

    def __create_pad_template_symbols(self, element, plugin_name):
        templates = element.get('pad-templates', {})
        if not templates:
            return

        for tname, template in templates.items():
            name = tname.replace("%%", "%")
            unique_name = '%s!%s' % (element['hierarchy'][0], name)
            pagename = 'element-' + element['name']
            object_type = self.__create_object_type(
                pagename, template.get("object-type"))
            self.create_symbol(GstPadTemplateSymbol,
                               name=name,
                               direction=template["direction"],
                               presence=template["presence"],
                               caps=template["caps"],
                               filename=plugin_name,
                               parent_name=None,
                               object_type=object_type,
                               display_name=name,
                               unique_name=unique_name,
                               extra={'gst-element-name': pagename})

    def _get_aliases(self, aliases):
        return [
            alias for alias in aliases
            if not self.app.database.get_symbol(alias)
        ]

    def __extract_feature_comment(self, feature_type, feature):
        pagename = feature_type + '-' + feature['name']
        possible_comment_names = [pagename, feature['name']]
        if feature_type == 'element':
            possible_comment_names.append(feature['hierarchy'][0])

        comment = None
        for comment_name in possible_comment_names:
            comment = self.app.database.get_comment(comment_name)
            if comment:
                break

        description = feature.get('description')
        if not comment:
            comment = Comment(
                pagename,
                Comment(description=feature['name']),
                description=description,
                short_description=Comment(description=description))
            self.app.database.add_comment(comment)
        elif not comment.short_description:
            comment.short_description = Comment(description=description)
        comment.title = Comment(description=feature['name'])
        comment.name = feature.get('name', pagename)
        comment.meta['title'] = feature['name']
        self.__toplevel_comments.add(comment)

        return pagename, comment

    def __parse_plugin(self, plugin_name, plugin):
        elements = []
        feature_names = list(plugin.get('elements', {}).keys()) \
            + list(plugin.get('tracers', {}).keys()) \
            + list(plugin.get('device-providers', {}).keys())

        if self.plugin and len(feature_names) == 1:
            self.unique_feature = feature_names[0]

        for ename, tracer in plugin.get('tracers', {}).items():
            tracer['name'] = ename
            self.__extract_feature_comment("tracer", tracer)

        for provider_name, provider in plugin.get('device-providers',
                                                  {}).items():
            provider['name'] = provider_name
            _, comment = self.__extract_feature_comment("provider", provider)
            comment.description += """\n\n# Provided device example"""
            self.__create_object_type(provider_name,
                                      provider.get('device-example'))

        for ename, element in plugin.get('elements', {}).items():
            element['name'] = ename
            pagename, _ = self.__extract_feature_comment("element", element)

            aliases = self._get_aliases([pagename, element['hierarchy'][0]])
            sym = self.create_symbol(GstElementSymbol,
                                     parent_name=None,
                                     display_name=element['name'],
                                     hierarchy=create_hierarchy(element),
                                     unique_name=element['name'],
                                     filename=plugin_name,
                                     extra={'gst-element-name': pagename},
                                     rank=str(element['rank']),
                                     author=element['author'],
                                     classification=element['klass'],
                                     plugin=plugin['filename'],
                                     aliases=aliases,
                                     package=plugin['package'])

            if not sym:
                continue

            self.__elements[element['name']] = sym
            self.__create_property_symbols(element, element['name'], pagename)
            self.__create_signal_symbols(element, element['name'], pagename)
            self.__create_pad_template_symbols(element, plugin_name)

            elements.append(sym)

        plugin = self.create_symbol(
            GstPluginSymbol,
            description=plugin['description'],
            display_name=plugin_name,
            unique_name='plugin-' + plugin['filename'],
            license=plugin['license'],
            package=plugin['package'],
            filename=plugin['filename'],
            elements=elements,
            extra={'gst-plugins': 'plugins-' + plugin['filename']})

        if not plugin:
            return None

        self.__all_plugins_symbols.add(plugin)

        if self.plugin:
            self.__plugins = plugin
        return plugin

    def __get_link_cb(self, resolver, name):
        link = self.__dual_links.get(name)
        if link:
            # Upsert link on the first run
            if isinstance(link, str):
                sym = self.app.database.get_symbol(link)
                link = sym.link

        if link:
            return link

        return (resolver, name)

    def format_page(self, page, link_resolver, output):
        link_resolver.get_link_signal.connect(GIExtension.search_online_links)
        super().format_page(page, link_resolver, output)
        link_resolver.get_link_signal.disconnect(
            GIExtension.search_online_links)
Пример #13
0
class Page:
    "Banana banana"
    meta_schema = {Optional('title'): And(str, len),
                   Optional('short-description'): And(str),
                   Optional('description'): And(str),
                   Optional('render-subpages'): bool,
                   Optional('auto-sort'): bool,
                   Optional('full-width'): bool,
                   Optional('see-also'): And(str, len),
                   Optional('extra'): Schema({str: object}),
                   Optional('thumbnail'): And(str, len),
                   Optional('include'): And(str, len),
                   Optional('redirect'): str,
                   }

    # pylint: disable=too-many-arguments
    # pylint: disable=too-many-locals
    def __init__(self, name, generated, project_name, extension_name,
                 source_file=None,
                 ast=None,
                 output_path='',
                 raw_contents=None,
                 comment=None,
                 meta=None,
                 pre_sorted=False,
                 symbol_names=None):
        assert name

        if not generated:
            assert source_file is not None

        self.name = name
        basename = os.path.basename(name)
        name = os.path.splitext(basename)[0]
        ref = os.path.join(output_path,
                           re.sub(r'\W+', '-', os.path.splitext(basename)[0]))
        pagename = '%s.html' % ref

        self.generated = generated
        self.project_name = project_name
        self.extension_name = extension_name
        self.source_file = source_file
        self.ast = ast
        self.raw_contents = raw_contents
        self.comment = comment
        self.pre_sorted = pre_sorted
        self.symbol_names = OrderedSet(symbol_names or [])

        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.private_symbols = []
        self.typed_symbols = OrderedDict()
        self.by_parent_symbols = OrderedDict()
        self.formatted_contents = None
        self.detailed_description = None
        self.build_path = None
        self.cached_paths = OrderedSet()

        if comment:
            meta = comment.meta
        elif meta:
            meta = meta
        else:
            meta = {}

        self.meta = {}
        for key, value in meta.items():
            try:
                self.meta.update(Schema(Page.meta_schema).validate({
                    key.replace('_', '-').lower(): value}))
            except SchemaError as err:
                warn('invalid-page-metadata',
                     '%s: Invalid metadata: \n%s, discarding metadata' %
                     (self.name, str(err)))

        if not self.meta.get('extra'):
            self.meta['extra'] = defaultdict()

        self.title = self.meta.get(
            'title', cmark.title_from_ast(self.ast) if ast else '')
        self.thumbnail = self.meta.get('thumbnail')
        self.short_description = self.meta.get('short-description', None)
        self.render_subpages = self.meta.get('render-subpages', True)

        self.link = Link(pagename, self.title or name, ref)

    def __repr__(self):
        return "<Page %s>" % self.name

    @staticmethod
    def __get_empty_typed_symbols():
        typed_symbols_list = namedtuple(
            'TypedSymbolsList', ['name', 'symbols'])
        empty_typed_symbols = {}

        for subclass in all_subclasses(Symbol):
            empty_typed_symbols[subclass] = typed_symbols_list(
                subclass.get_plural_name(), [])

        return empty_typed_symbols

    def resolve_symbols(self, tree, database, link_resolver):
        """
        When this method is called, the page's symbol names are queried
        from `database`, and added to lists of actual symbols, sorted
        by symbol class.
        """
        self.typed_symbols = self.__get_empty_typed_symbols()
        all_syms = OrderedSet()
        for sym_name in self.symbol_names:
            sym = database.get_symbol(sym_name)
            self.__query_extra_symbols(
                sym, all_syms, tree, link_resolver, database)

        if tree.project.is_toplevel:
            page_path = self.link.ref
        else:
            page_path = self.project_name + '/' + self.link.ref

        if self.meta.get("auto-sort", True):
            all_syms = sorted(all_syms, key=lambda x: x.unique_name)
        for sym in all_syms:
            sym.update_children_comments()
            self.__resolve_symbol(sym, link_resolver, page_path)
            self.symbol_names.add(sym.unique_name)

        # Always put symbols with no parent at the end
        no_parent_syms = self.by_parent_symbols.pop(None, None)
        if no_parent_syms:
            self.by_parent_symbols[None] = no_parent_syms

        for sym_type in [ClassSymbol, AliasSymbol, InterfaceSymbol,
                         StructSymbol]:
            syms = self.typed_symbols[sym_type].symbols

            if not syms:
                continue

            if self.title is None:
                self.title = syms[0].display_name
            if self.comment is None:
                self.comment = Comment(name=self.name)
                self.comment.short_description = syms[
                    0].comment.short_description
                self.comment.title = syms[0].comment.title
            break

    # pylint: disable=no-self-use
    def __fetch_comment(self, sym, database):
        sym.comment = database.get_comment(
            sym.unique_name) or Comment(sym.unique_name)

        for sym in sym.get_children_symbols():
            if isinstance(sym, Symbol):
                self.__fetch_comment(sym, database)

    def __format_page_comment(self, formatter, link_resolver):
        if not self.comment:
            return

        if self.comment.short_description:
            self.short_description = formatter.format_comment(
                self.comment.short_description, link_resolver).strip()
            if self.short_description.startswith('<p>'):
                self.short_description = self.short_description[3:-4]
        if self.comment.title:
            self.title = formatter.format_comment(
                self.comment.title, link_resolver).strip()
            if self.title.startswith('<p>'):
                self.title = self.title[3:-4]

        if self.title:
            self.formatted_contents += '<h1 id="%s-page">%s</h1>' % (
                id_from_text(self.title), self.title)

        self.formatted_contents += formatter.format_comment(
            self.comment, link_resolver)

    def __redirect_if_needed(self, formatter, link_resolver):
        redirect = self.meta.get("redirect")
        if not redirect:
            return False
        link = link_resolver.get_named_link(redirect)
        if link:
            if formatter.extension.project.is_toplevel:
                page_path = self.link.ref
            else:
                page_path = self.project_name + '/' + self.link.ref
            self_dir = os.path.dirname(page_path)
            if self_dir == self.link.ref:
                self_dir = ""
            self.meta["redirect"] = os.path.relpath(link.ref, self_dir)
        else:
            warn('markdown-bad-link', "Bad redirect link '%s' in page: %s"
                 % (redirect, self.name))
        return True

    def __format_content(self, formatter, link_resolver):
        if self.ast:
            out, diags = cmark.ast_to_html(self.ast, link_resolver)
            for diag in diags:
                warn(
                    diag.code,
                    message=diag.message,
                    filename=self.source_file or self.name)
            self.formatted_contents += out

        if not self.formatted_contents:
            self.__format_page_comment(formatter, link_resolver)

    def format(self, formatter, link_resolver, output):
        """
        Banana banana
        """

        if not self.title and self.name:
            title = os.path.splitext(self.name)[0]
            self.title = os.path.basename(title).replace('-', ' ')

        self.formatted_contents = u''

        self.build_path = os.path.join(formatter.get_output_folder(self),
                                       self.link.ref)
        if not self.__redirect_if_needed(formatter, link_resolver):
            self.__format_content(formatter, link_resolver)

        self.output_attrs = defaultdict(lambda: defaultdict(dict))
        formatter.prepare_page_attributes(self)
        self.__format_symbols(formatter, link_resolver)
        self.detailed_description =\
            formatter.format_page(self)[0]

        if output:
            formatter.cache_page(self)

    # pylint: disable=no-self-use
    def get_title(self):
        """
        Banana banana
        """
        return self.title or 'unnamed'

    def __discover_title(self, meta):
        if meta is not None and 'title' in meta:
            self.title = meta['title']
        elif self.ast:
            self.title = cmark.title_from_ast(self.ast)

    def __format_symbols(self, formatter, link_resolver):
        for symbol in self.symbols:
            if symbol is None:
                continue
            debug('Formatting symbol %s in page %s' % (
                symbol.unique_name, self.name), 'formatting')
            symbol.detailed_description = formatter.format_symbol(
                symbol, link_resolver)

    def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
                              database):
        if sym:
            self.__fetch_comment(sym, database)
            all_syms.add(sym)

    def __resolve_symbol(self, symbol, link_resolver, page_path):
        symbol.resolve_links(link_resolver)

        symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)

        for link in symbol.get_extra_links():
            link.ref = "%s#%s" % (page_path, link.id_)

        tsl = self.typed_symbols.get(type(symbol))
        if tsl:
            tsl.symbols.append(symbol)

            by_parent_symbols = self.by_parent_symbols.get(symbol.parent_name)
            if not by_parent_symbols:
                by_parent_symbols = self.__get_empty_typed_symbols()
                parent_name = symbol.parent_name
                if parent_name is None:
                    parent_name = 'Others symbols'
                self.by_parent_symbols[symbol.parent_name] = by_parent_symbols
            by_parent_symbols.get(type(symbol)).symbols.append(symbol)

        self.symbols.append(symbol)

        debug('Resolved symbol %s to page %s' %
              (symbol.unique_name, self.link.ref), 'resolution')
Пример #14
0
class GstExtension(Extension):
    extension_name = 'gst-extension'
    argument_prefix = 'gst'
    __dual_links = {}  # Maps myelement:XXX to GstMyElement:XXX
    __parsed_cfiles = set()
    __caches = {}  # cachefile -> dict
    __all_plugins_symbols = set()

    def __init__(self, app, project):
        super().__init__(app, project)
        self.cache = {}
        self.c_sources = []
        self.cache_file = None
        self.plugin = None
        self.__elements = {}
        self.__raw_comment_parser = GtkDocParser(project,
                                                 section_file_matching=False)
        self.__plugins = None
        self.__toplevel_comments = OrderedSet()
        self.list_plugins_page = None
        # If we have a plugin with only one element, we render it on the plugin
        # page.
        self.unique_feature = None
        self.__on_index_symbols = []

        # Links GTypeName to pagename for other-types so we now where to locate
        # the symbols on creation.
        self.__other_types_pages = {}

    def _make_formatter(self):
        return GstFormatter(self)

    def create_symbol(self, *args, **kwargs):
        sym = super().create_symbol(*args, **kwargs)
        if self.unique_feature and sym:
            self.__on_index_symbols.append(sym)

        return sym

    # pylint: disable=too-many-branches
    def setup(self):
        gi_extension = self.project.extensions.get('gi-extension')
        self.gi_languages = [
            lang.language_name for lang in gi_extension.languages
        ]

        if not self.cache_file:
            if self.list_plugins_page:
                self.__plugins = self.create_symbol(
                    GstPluginsSymbol,
                    display_name="All " +
                    self.project.project_name.replace("-", " ").title(),
                    unique_name=self.project.project_name + "-all-gst-plugins",
                    plugins=[],
                    all_plugins=True)

                super().setup()
            return

        gather_links()

        comment_parser = GtkDocParser(self.project, False)
        to_parse_sources = set(self.c_sources) - GstExtension.__parsed_cfiles

        CCommentExtractor(self,
                          comment_parser).parse_comments(to_parse_sources)
        GstExtension.__parsed_cfiles.update(self.c_sources)

        self.debug("Parsing plugin %s, (cache file %s)" %
                   (self.plugin, self.cache_file))

        if not self.cache:
            error('setup-issue',
                  "No cache loaded or created for %s" % self.plugin)

        plugins = []
        if self.plugin:
            pname = self.plugin
            dot_idx = pname.rfind('.')
            if dot_idx > 0:
                pname = self.plugin[:dot_idx]
            if pname.startswith('libgst'):
                pname = pname[6:]
            elif pname.startswith('gst'):
                pname = pname[3:]
            try:
                plugin_node = {pname: self.cache[pname]}
            except KeyError:
                error('setup-issue', "Plugin %s not found" % pname)
        else:
            plugin_node = self.cache

        for libfile, plugin in plugin_node.items():
            plugin_sym = self.__parse_plugin(libfile, plugin)
            if not plugin_sym:
                continue

            plugins.append(plugin_sym)

        if not self.plugin:
            self.__plugins = self.create_symbol(
                GstPluginsSymbol,
                display_name=self.project.project_name.replace("-",
                                                               " ").title(),
                unique_name=self.project.project_name + "-gst-plugins",
                plugins=plugins)

        super().setup()

    def _get_comment_smart_key(self, comment):
        try:
            return comment.title.description
        except AttributeError:
            return None

    def _get_toplevel_comments(self):
        if self.unique_feature:
            return OrderedSet()
        return self.__toplevel_comments

    def get_plugin_comment(self):
        if self.plugin:
            res = self.app.database.get_comment("plugin-" + self.plugin)
            return res
        return None

    def make_pages(self):
        smart_pages = super().make_pages()

        if not self.__plugins:
            return None

        if self.list_plugins_page is None:
            index = smart_pages.get('gst-index')
            if index is None:
                return smart_pages

            index.render_subpages = False
            index.symbol_names.add(self.__plugins.unique_name)
            for sym in self.__on_index_symbols:
                index.symbol_names.add(sym.unique_name)
            if self.unique_feature:
                index.comment = self.app.database.get_comment(
                    "element-" + self.unique_feature)
            else:
                index.comment = self.get_plugin_comment()
            return smart_pages

        page = smart_pages.get(self.list_plugins_page)
        page.render_subpages = False
        page.extension_name = self.extension_name

        page.symbol_names.add(self.__plugins.unique_name)
        self.__plugins.plugins = self.__all_plugins_symbols

        return smart_pages

    def add_comment(self, comment):
        # We handle toplevel comments ourself, make sure all comments
        # end up in the database
        comment.toplevel = False

        super().add_comment(comment)

    def _get_smart_index_title(self):
        if self.plugin:
            return self.__plugins.display_name
        return 'GStreamer plugins documentation'

    @staticmethod
    def add_arguments(parser):
        group = parser.add_argument_group('Gst extension', DESCRIPTION)
        GstExtension.add_index_argument(group)
        # GstExtension.add_order_generated_subpages(group)
        GstExtension.add_sources_argument(group, prefix='gst-c')
        group.add_argument('--gst-cache-file', default=None)
        group.add_argument('--gst-list-plugins-page', default=None)
        group.add_argument('--gst-plugin-name', default=None)
        group.add_argument('--gst-plugins-path', default=None)

    def parse_config(self, config):
        self.c_sources = config.get_sources('gst_c')
        self.cache_file = config.get('gst_cache_file')
        self.plugin = config.get('gst_plugin_name')
        self.list_plugins_page = config.get('gst_list_plugins_page', None)
        info('Parsing config!')

        self.cache = {}
        if self.cache_file:
            self.cache = GstExtension.__caches.get(self.cache_file)
            if not self.cache:
                try:
                    with open(self.cache_file) as _:
                        self.cache = json.load(_)
                except FileNotFoundError:
                    pass

                if self.cache is None:
                    self.cache = {}
                GstExtension.__caches[self.cache_file] = self.cache

        super().parse_config(config)

    def _get_smart_key(self, symbol):
        if self.unique_feature:
            return None

        if isinstance(symbol, GstPluginSymbol):
            # PluginSymbol are rendered on the index page
            return None
        res = symbol.extra.get('gst-element-name')
        if res:
            res = res.replace("element-", "")

        return res

    def __create_symbol(self, gtype, symbol, pagename):
        if symbol["kind"] in ["enum", "flags"]:
            if pagename in self.__other_types_pages:
                # The enum was defined by another type (not an element)
                # this page will be rendered as any GObject so we need
                # to specify its parent name
                parent_name = pagename
            else:
                parent_name = None
            return self.__create_enum_symbol(gtype,
                                             symbol.get('values'),
                                             pagename,
                                             parent_name=parent_name)
        if symbol["kind"] == "object":
            return self.__create_classed_type(gtype, symbol)
        if symbol["kind"] == "interface":
            return self.__create_classed_type(gtype, symbol, True)

        assert "Not reached" == "False"
        return None

    def _remember_symbol_type(self, gtype, pagename):
        self.__other_types_pages[gtype] = pagename

        return gtype

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-arguments
    def __create_signal_symbol(self,
                               obj,
                               parent_uniquename,
                               name,
                               signal,
                               pagename,
                               parent_name=None):
        args = signal['args']
        instance_type = obj['hierarchy'][0]
        unique_name = "%s::%s" % (parent_uniquename, name)
        aliases = ["%s::%s" % (instance_type, name)]

        gi_extension = self.project.extensions.get('gi-extension')
        python_lang = gi_extension.get_language('python')
        args_type_names = [
            (type_tokens_from_type_name('GstElement', python_lang), 'param_0')
        ]
        for arg in args:
            arg_name = arg["name"]
            arg_type_name = self._remember_symbol_type(arg["type"], pagename)
            type_tokens = type_tokens_from_type_name(arg_type_name,
                                                     python_lang)
            args_type_names.append((type_tokens, arg_name))

        if not signal.get('action'):
            args_type_names.append(
                (type_tokens_from_type_name('gpointer', python_lang), "udata"))
        params = []

        for comment_name in [unique_name] + aliases:
            comment = self.app.database.get_comment(comment_name)
            if comment:
                for i, argname in enumerate(comment.params.keys()):
                    args_type_names[i] = (args_type_names[i][0], argname)

        for tokens, argname in args_type_names:
            params.append(ParameterSymbol(argname=argname, type_tokens=tokens))

        return_type_name = self._remember_symbol_type(signal["return-type"],
                                                      pagename)
        if return_type_name == 'void':
            retval = [None]
        else:
            tokens = type_tokens_from_type_name(return_type_name, python_lang)
            retval = [ReturnItemSymbol(type_tokens=tokens)]

        if signal.get('action'):
            typ = ActionSignalSymbol
            for param in params:
                self.add_attrs(param, action=True)
        else:
            typ = SignalSymbol
        res = self.create_symbol(typ,
                                 parameters=params,
                                 return_value=retval,
                                 display_name=name,
                                 unique_name=unique_name,
                                 extra={'gst-element-name': pagename},
                                 aliases=aliases,
                                 parent_name=parent_name)

        if res:
            flags = []

            when = signal.get('when')
            if when == "first":
                flags.append(gi.flags.RunFirstFlag())
            elif when == "last":
                flags.append(gi.flags.RunLastFlag())
            elif when == "cleanup":
                flags.append(gi.flags.RunCleanupFlag())

            no_hooks = signal.get('no-hooks')
            if no_hooks:
                flags.append(gi.flags.NoHooksFlag())

            action = signal.get('action')
            if action:
                flags.append(gi.flags.ActionFlag())

            # This is incorrect, it's not yet format time
            extra_content = self.formatter.format_flags(flags)
            res.extension_contents['Flags'] = extra_content

        return res

    def __create_signal_symbols(self,
                                obj,
                                parent_uniquename,
                                element_name,
                                parent_name=None):
        res = []
        signals = obj.get('signals', {})
        if not signals:
            return res

        for name, signal in signals.items():
            res.append(
                self.__create_signal_symbol(obj,
                                            parent_uniquename,
                                            name,
                                            signal,
                                            element_name,
                                            parent_name=parent_name))
        return res

    def __create_property_symbols(self,
                                  obj,
                                  parent_uniquename,
                                  pagename,
                                  parent_name=None):
        res = []
        properties = obj.get('properties', [])
        if not properties:
            return res

        gi_extension = self.project.extensions.get('gi-extension')
        python_lang = gi_extension.get_language('python')
        for name, prop in properties.items():
            unique_name = '%s:%s' % (obj.get('name', parent_uniquename), name)
            flags = [ReadableFlag()]
            if prop['writable']:
                flags += [WritableFlag()]
            if prop['construct-only']:
                flags += [ConstructOnlyFlag()]
            elif prop['construct']:
                flags += [ConstructFlag()]

            prop_type_name = self._remember_symbol_type(prop["type"], pagename)

            tokens = type_tokens_from_type_name(prop_type_name, python_lang)
            type_ = QualifiedSymbol(type_tokens=tokens)

            default = prop.get('default')
            if obj['hierarchy'][0] != parent_uniquename:
                aliases = ['%s:%s' % (obj['hierarchy'][0], name)]
            else:
                aliases = []

            sym = self.app.database.get_symbol(unique_name)
            if sym is None:
                sym = self.create_symbol(
                    PropertySymbol,
                    prop_type=type_,
                    display_name=name,
                    unique_name=unique_name,
                    aliases=aliases,
                    parent_name=parent_name,
                    extra={'gst-element-name': pagename},
                )
            assert sym

            if not self.app.database.get_comment(unique_name):
                comment = Comment(unique_name,
                                  Comment(name=name),
                                  description=prop['blurb'])
                self.app.database.add_comment(comment)

            # FIXME This is incorrect, it's not yet format time (from gi_extension)
            extra_content = self.formatter.format_flags(flags)
            sym.extension_contents['Flags'] = extra_content
            if default:
                if prop_type_name in ['GstCaps', 'GstStructure']:
                    default = '<pre class="language-yaml">' + \
                        '<code class="language-yaml">%s</code></pre>' % default
                sym.extension_contents['Default value'] = default
            res.append(sym)

        return res

    def __create_enum_symbol(self,
                             type_name,
                             enum,
                             pagename,
                             parent_name=None):
        display_name = re.sub(r"([a-z])([A-Z])", r"\g<1>-\g<2>",
                              type_name.replace('Gst', ''))

        unique_name = type_name
        if self.app.database.get_symbol(unique_name):
            # Still required as some bin manually proxy children properties inside
            # themselves (like GstFakeSinkStateError in fakevideosink for example)
            unique_name = pagename + '_' + type_name
        members = []
        for val in enum:
            value_unique_name = "%s::%s" % (type_name, val['name'])
            if self.app.database.get_symbol(value_unique_name):
                value_unique_name = "%s_%s" % (pagename, value_unique_name)
            member_sym = self.create_symbol(
                GstNamedConstantValue,
                unique_name=value_unique_name,
                display_name=val['name'],
                value=val['value'],
                parent_name=parent_name,
                val=val,
                extra={'gst-element-name': pagename})
            if member_sym:
                members.append(member_sym)

        symbol = self.create_symbol(GstNamedConstantsSymbols,
                                    anonymous=False,
                                    raw_text=None,
                                    display_name=display_name.capitalize(),
                                    unique_name=unique_name,
                                    parent_name=parent_name,
                                    members=members,
                                    extra={'gst-element-name': pagename})

        if not symbol:
            return None

        symbol.values = enum
        return symbol

    def __create_hierarchy(self, pagename, element_dict):
        hierarchy = []
        for klass_name in element_dict["hierarchy"][1:]:
            self._remember_symbol_type(klass_name, pagename)
            link = Link(None, klass_name, klass_name, mandatory=True)
            sym = QualifiedSymbol(type_tokens=[link])
            hierarchy.append(sym)

        hierarchy.reverse()
        return hierarchy

    def __create_classed_type(self, pagename, _object, is_interface=False):
        unique_name = _object['hierarchy'][0]
        properties = self.__create_property_symbols(_object,
                                                    unique_name,
                                                    pagename,
                                                    parent_name=unique_name)
        signals = self.__create_signal_symbols(_object,
                                               unique_name,
                                               pagename,
                                               parent_name=unique_name)

        return self.create_symbol(
            GIInterfaceSymbol if is_interface else GIClassSymbol,
            hierarchy=self.__create_hierarchy(pagename, _object),
            display_name=unique_name,
            unique_name=unique_name,
            parent_name=unique_name,
            properties=properties,
            signals=signals,
            extra={'gst-element-name': pagename})

    def __create_pad_template_symbols(self, element, plugin_name):
        templates = element.get('pad-templates', {})
        res = []
        if not templates:
            return res

        for tname, template in templates.items():
            name = tname.replace("%%", "%")
            unique_name = '%s!%s' % (element['hierarchy'][0], name)
            pagename = 'element-' + element['name']
            gtype = self._remember_symbol_type(template.get("type", "GstPad"),
                                               pagename)
            link = Link(None, gtype, gtype, mandatory=True)
            object_type = QualifiedSymbol(type_tokens=[link])
            res.append(
                self.create_symbol(GstPadTemplateSymbol,
                                   name=name,
                                   direction=template["direction"],
                                   presence=template["presence"],
                                   caps=template["caps"],
                                   filename=plugin_name,
                                   parent_name=None,
                                   object_type=object_type,
                                   display_name=name,
                                   unique_name=unique_name,
                                   extra={'gst-element-name': pagename}))

        return res

    def __extract_feature_comment(self, feature_type, feature):
        pagename = feature_type + '-' + feature['name']
        possible_comment_names = [pagename, feature['name']]
        if feature_type == 'element':
            possible_comment_names.append(feature['hierarchy'][0])

        comment = None
        for comment_name in possible_comment_names:
            comment = self.app.database.get_comment(comment_name)
            if comment:
                break

        description = feature.get('description')
        if not comment:
            comment = Comment(
                pagename,
                Comment(description=feature['name']),
                description=description,
                short_description=Comment(description=description))
            self.app.database.add_comment(comment)
        elif not comment.short_description:
            comment.short_description = Comment(description=description)
        comment.title = Comment(description=feature['name'])
        comment.name = feature.get('name', pagename)
        comment.meta['title'] = feature['name']
        self.__toplevel_comments.add(comment)

        return pagename, comment

    def __parse_plugin(self, plugin_name, plugin):
        elements = []
        feature_names = list(plugin.get('elements', {}).keys()) \
            + list(plugin.get('tracers', {}).keys()) \
            + list(plugin.get('device-providers', {}).keys())

        self.__other_types_pages = {}
        if self.plugin and len(feature_names) == 1:
            self.unique_feature = feature_names[0]

        for ename, tracer in plugin.get('tracers', {}).items():
            tracer['name'] = ename
            self.__extract_feature_comment("tracer", tracer)

        other_types = []
        for provider_name, provider in plugin.get('device-providers',
                                                  {}).items():
            provider['name'] = provider_name
            _, comment = self.__extract_feature_comment("provider", provider)
            comment.description += """\n\n# Provided device example"""
            other_types.append(
                self.__create_classed_type(provider_name,
                                           provider.get('device-example')))

        for ename, element in plugin.get('elements', {}).items():
            element['name'] = ename
            pagename, _ = self.__extract_feature_comment("element", element)
            interfaces = []
            for interface in element.get("interfaces", []):
                self._remember_symbol_type(interface, pagename)
                interfaces.append(
                    QualifiedSymbol(type_tokens=[
                        Link(None, interface, interface, mandatory=True)
                    ]))

            aliases = [pagename, element['hierarchy'][0]]
            sym = self.create_symbol(GstElementSymbol,
                                     parent_name=None,
                                     display_name=element['name'],
                                     hierarchy=self.__create_hierarchy(
                                         pagename, element),
                                     unique_name=element['name'],
                                     filename=plugin_name,
                                     extra={'gst-element-name': pagename},
                                     rank=str(element['rank']),
                                     author=element['author'],
                                     classification=element['klass'],
                                     plugin=plugin_name,
                                     aliases=aliases,
                                     package=plugin['package'],
                                     interfaces=interfaces)

            if not sym:
                continue

            self.__elements[element['name']] = sym
            sym.properties.extend(
                self.__create_property_symbols(element, element['name'],
                                               pagename))
            sym.signals.extend(
                self.__create_signal_symbols(element, element['name'],
                                             pagename))
            sym.pad_templates.extend(
                self.__create_pad_template_symbols(element, plugin_name))

            elements.append(sym)

        types = list(plugin['other-types'].items())
        while True:
            type_ = None
            for tmptype in types:
                if tmptype[0] in self.__other_types_pages:
                    type_ = tmptype
                    break

            if not type_:
                break

            types.remove(type_)
            other_types.append(
                self.__create_symbol(type_[0], type_[1],
                                     self.__other_types_pages[type_[0]]))

        for _type in types:
            self.warn(
                "no-location-indication",
                "Type %s has been marked with `gst_type_mark_as_plugin_api`"
                " but is not used in any of %s API (it might require to"
                " be manually removed from the cache in case of plugin"
                " move)" % (_type[0], plugin_name))

        plugin = self.create_symbol(
            GstPluginSymbol,
            description=plugin['description'],
            display_name=plugin_name,
            unique_name='plugin-' + plugin_name,
            license=plugin['license'],
            package=plugin['package'],
            filename=plugin['filename'],
            elements=elements,
            other_types=other_types,
            extra={'gst-plugins': 'plugins-' + plugin['filename']})

        if not plugin:
            return None

        self.__all_plugins_symbols.add(plugin)

        if self.plugin:
            self.__plugins = plugin
        return plugin

    def __get_link_cb(self, resolver, name):
        link = self.__dual_links.get(name)
        if link:
            # Upsert link on the first run
            if isinstance(link, str):
                sym = self.app.database.get_symbol(link)
                link = sym.link

        if link:
            return link

        return (resolver, name)

    def format_page(self, page, link_resolver, output):
        link_resolver.get_link_signal.connect(search_online_links)
        gi_extension = self.project.extensions.get('gi-extension')
        page.meta['extra']['gi-languages'] = self.gi_languages
        super().format_page(page, link_resolver, output)
        link_resolver.get_link_signal.disconnect(search_online_links)
Пример #15
0
class TestTree(unittest.TestCase):
    def setUp(self):
        here = os.path.dirname(__file__)
        self.__md_dir = os.path.abspath(
            os.path.join(here, 'tmp-markdown-files'))
        self.private_folder = os.path.abspath(os.path.join(
            here, 'tmp-private'))
        self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
        self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
        self.__remove_tmp_dirs()
        os.mkdir(self.__md_dir)
        os.mkdir(self.private_folder)
        os.mkdir(self.__src_dir)
        os.mkdir(self.get_generated_doc_folder())
        self.include_paths = OrderedSet([self.__md_dir])
        self.include_paths.add(self.get_generated_doc_folder())

        # Using the real doc database is too costly, tests should be lightning
        # fast (and they are)
        self.database = Database()
        self.database.setup(self.private_folder)
        self.link_resolver = LinkResolver(self.database)

        self.change_tracker = ChangeTracker()

        self.sitemap_parser = SitemapParser()

        self.project_name = 'test-project'
        self.sanitized_name = 'test-project-0.1'
        self.incremental = False

        self.tree = Tree(self, self)

        self.test_ext = TestExtension(self, self)
        self.core_ext = CoreExtension(self, self)

        cfg = Config()

        self.test_ext.parse_config(cfg)
        self.core_ext.parse_config(cfg)
        self.subprojects = {}
        self.is_toplevel = True

    def tearDown(self):
        self.__remove_tmp_dirs()
        del self.test_ext
        del self.core_ext

    def get_generated_doc_folder(self):
        return os.path.join(self.private_folder, 'generated')

    def get_base_doc_folder(self):
        return self.__md_dir

    def get_private_folder(self):
        return self.private_folder

    def __parse_sitemap(self, text):
        path = os.path.join(self.__md_dir, 'sitemap.txt')
        with io.open(path, 'w', encoding='utf-8') as _:
            _.write(text)
        return self.sitemap_parser.parse(path)

    def __create_md_file(self, name, contents):
        path = os.path.join(self.__md_dir, name)
        with open(path, 'w') as _:
            _.write(contents)

        # Just making sure we don't hit a race condition,
        # in real world situations it is assumed users
        # will not update source files twice in the same
        # microsecond
        touch(path)

    def __create_src_file(self, name, symbols):
        path = os.path.join(self.__md_dir, name)
        with open(path, 'w') as _:
            for symbol in symbols:
                _.write('%s\n' % symbol)

        # Just making sure we don't hit a race condition,
        # in real world situations it is assumed users
        # will not update source files twice in the same
        # microsecond
        touch(path)

        return path

    def __remove_src_file(self, name):
        path = os.path.join(self.__md_dir, name)
        os.unlink(path)

    def __remove_md_file(self, name):
        path = os.path.join(self.__md_dir, name)
        os.unlink(path)

    def __touch_src_file(self, name):
        path = os.path.join(self.__md_dir, name)
        touch(path)

    def __remove_tmp_dirs(self):
        shutil.rmtree(self.__md_dir, ignore_errors=True)
        shutil.rmtree(self.private_folder, ignore_errors=True)
        shutil.rmtree(self.__src_dir, ignore_errors=True)
        shutil.rmtree(self.__output_dir, ignore_errors=True)
        shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)

    def test_basic(self):
        inp = (u'index.markdown\n' '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('section.markdown', (u'# My section\n'))

        self.tree.parse_sitemap(sitemap)

        pages = self.tree.get_pages()

        # We do not care about ordering
        self.assertSetEqual(set(pages.keys()),
                            set([u'index.markdown', u'section.markdown']))

        index = pages.get('index.markdown')
        self.assertEqual(index.title, u'My documentation')

    def test_basic_incremental(self):
        inp = (u'index.markdown\n' '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('section.markdown', (u'# My section\n'))

        self.tree.parse_sitemap(sitemap)

        # Building from scratch, all pages are stale
        self.assertSetEqual(set(self.tree.get_stale_pages()),
                            set([u'index.markdown', u'section.markdown']))

        self.tree.persist()

        self.incremental = True
        self.tree = Tree(self, self)
        self.tree.parse_sitemap(sitemap)

        # Nothing changed, no page is stale
        self.assertSetEqual(set(self.tree.get_stale_pages()), set({}))

        # But we still have our pages
        self.assertSetEqual(set(self.tree.get_pages()),
                            set([u'index.markdown', u'section.markdown']))

        touch(os.path.join(self.__md_dir, u'section.markdown'))

        self.tree = Tree(self, self)
        self.tree.parse_sitemap(sitemap)

        self.assertSetEqual(set(self.tree.get_stale_pages()),
                            set([u'section.markdown']))

    def __assert_extension_names(self, tree, name_map):
        pages = tree.get_pages()
        for name, ext_name in list(name_map.items()):
            page = pages[name]
            self.assertEqual(ext_name, page.extension_name)

    def __assert_stale(self, expected_stale):
        stale_pages = self.tree.get_stale_pages()
        for pagename in expected_stale:
            self.assertIn(pagename, stale_pages)
            stale_pages.pop(pagename)
        self.assertEqual(len(stale_pages), 0)

    def __create_test_layout(self, with_ext_index=True, sitemap=None):
        if not sitemap:
            inp = (u'index.markdown\n'
                   '\ttest-index\n'
                   '\t\ttest-section.markdown\n'
                   '\t\t\tsource_a.test\n'
                   '\t\tpage_x.markdown\n'
                   '\t\tpage_y.markdown\n'
                   '\tcore_page.markdown\n')
        else:
            inp = sitemap

        sources = []

        sources.append(
            self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2']))

        sources.append(
            self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4']))

        if with_ext_index:
            self.test_ext.index = 'test-index.markdown'

        self.test_ext.sources = sources
        self.test_ext.setup()

        sitemap = self.__parse_sitemap(inp)

        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('core_page.markdown',
                              (u'# My non-extension page\n'))
        if with_ext_index:
            self.__create_md_file('test-index.markdown',
                                  (u'# My test index\n'))
        self.__create_md_file(
            'test-section.markdown',
            (u'# My test section\n'
             '\n'
             'Linking to [a generated page](source_a.test)\n'))
        self.__create_md_file('page_x.markdown', (u'---\n'
                                                  'symbols: [symbol_3]\n'
                                                  '...\n'
                                                  '# Page X\n'))
        self.__create_md_file('page_y.markdown', (u'# Page Y\n'))

        self.tree.parse_sitemap(sitemap)

        return sitemap

    def __update_test_layout(self, sitemap):
        self.test_ext.reset()
        self.tree = Tree(self, self)
        self.test_ext.setup()
        self.tree.parse_sitemap(sitemap)

    def test_extension_basic(self):
        _ = self.__create_test_layout()
        self.__assert_extension_names(
            self.tree, {
                u'index.markdown': 'core',
                u'test-index': 'test-extension',
                u'test-section.markdown': 'test-extension',
                u'source_a.test': 'test-extension',
                u'source_b.test': 'test-extension',
                u'page_x.markdown': 'test-extension',
                u'page_y.markdown': 'test-extension',
                u'core_page.markdown': 'core'
            })

        all_pages = self.tree.get_pages()
        self.assertEqual(len(all_pages), 8)
        self.__assert_stale(all_pages)
        self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
        self.assertIn('source_a.test',
                      all_pages['test-section.markdown'].subpages)

    def test_extension_override(self):
        self.__create_md_file('source_a.test.markdown', (u'# My override\n'))
        _ = self.__create_test_layout()
        page = self.tree.get_pages()['source_a.test']

        self.assertEqual(page.symbol_names,
                         OrderedSet(['symbol_1', 'symbol_2']))

        self.assertEqual(os.path.basename(page.source_file),
                         'source_a.test.markdown')

        out, _ = cmark.ast_to_html(page.ast, None)

        self.assertEqual(out, u'<h1>My override</h1>\n')

    def test_no_extension_index_override(self):
        _ = self.__create_test_layout(with_ext_index=False)
        ext_index = self.tree.get_pages()['test-index']
        self.assertEqual(ext_index.generated, True)
        self.assertEqual(len(ext_index.subpages), 4)

    def test_parse_yaml(self):
        inp = (u'index.markdown\n')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown',
                              (u'---\n'
                               'title: A random title\n'
                               'symbols: [symbol_1, symbol_2]\n'
                               '...\n'
                               '# My documentation\n'))

        self.tree.parse_sitemap(sitemap)

        pages = self.tree.get_pages()
        page = pages.get('index.markdown')

        out, _ = cmark.ast_to_html(page.ast, None)

        self.assertEqual(out, u'<h1>My documentation</h1>\n')

        self.assertEqual(page.title, u'A random title')

        self.assertEqual(page.symbol_names,
                         OrderedSet(['symbol_1', 'symbol_2']))

    def test_empty_link_resolution(self):
        inp = (u'index.markdown\n' '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('section.markdown', (u'# My section\n'
                                                   '\n'
                                                   '[](index.markdown)\n'))

        self.tree.parse_sitemap(sitemap)
        self.tree.resolve_symbols(self.database, self.link_resolver)
        self.tree.format(self.link_resolver, self.__output_dir,
                         {self.core_ext.extension_name: self.core_ext})

        pages = self.tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents, u'<h1>My section</h1>\n'
            '<p><a href="index.html">My documentation</a></p>\n')

    def test_labeled_link_resolution(self):
        inp = (u'index.markdown\n' '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('section.markdown',
                              (u'# My section\n'
                               '\n'
                               '[a label](index.markdown)\n'))

        self.tree.parse_sitemap(sitemap)
        self.tree.resolve_symbols(self.database, self.link_resolver)
        self.tree.format(self.link_resolver, self.__output_dir,
                         {self.core_ext.extension_name: self.core_ext})

        pages = self.tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents, u'<h1>My section</h1>\n'
            '<p><a href="index.html">a label</a></p>\n')

    def test_anchored_link_resolution(self):
        inp = (u'index.markdown\n' '\tsection.markdown')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('index.markdown', (u'# My documentation\n'))
        self.__create_md_file('section.markdown',
                              (u'# My section\n'
                               '\n'
                               '[](index.markdown#subsection)\n'))

        self.tree.parse_sitemap(sitemap)
        self.tree.resolve_symbols(self.database, self.link_resolver)
        self.tree.format(self.link_resolver, self.__output_dir,
                         {self.core_ext.extension_name: self.core_ext})

        pages = self.tree.get_pages()
        page = pages.get('section.markdown')
        self.assertEqual(
            page.formatted_contents, u'<h1>My section</h1>\n'
            '<p><a href="index.html#subsection">My documentation</a></p>\n')

    # pylint: disable=too-many-statements
    def test_extension_incremental(self):
        sitemap = self.__create_test_layout()
        self.tree.persist()

        self.incremental = True

        # Here we touch source_a.test, as its symbols were
        # all contained in a generated page, only that page
        # should now be stale
        self.__touch_src_file('source_a.test')
        self.__update_test_layout(sitemap)
        self.__assert_stale(set(['source_a.test']))
        self.tree.persist()

        # We now touch source_b.test, which symbols are contained
        # both in a generated page and a user-provided one.
        # We expect both pages to be stale
        self.__touch_src_file('source_b.test')
        self.__update_test_layout(sitemap)
        self.__assert_stale(set(['source_b.test', 'page_x.markdown']))
        self.tree.persist()

        # This one is trickier: we unlist symbol_3 from
        # page_x, which means the symbol should now be
        # documented in the generated page for source_b.test.
        # We expect both pages to be stale, and make sure
        # they contain the right symbols
        self.__create_md_file('page_x.markdown', (u'# Page X\n'))
        self.__update_test_layout(sitemap)
        self.__assert_stale(set(['source_b.test', 'page_x.markdown']))

        page_x = self.tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet())

        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names,
                         OrderedSet(['symbol_4', 'symbol_3']))

        self.tree.persist()

        # Let's make sure the opposite use case works as well,
        # we relocate symbol_3 in page_x , both page_x and
        # the generated page for source_b.test should be stale
        # and the symbols should be back to their original
        # layout.
        self.__create_md_file('page_x.markdown', (u'---\n'
                                                  'symbols: [symbol_3]\n'
                                                  '...\n'
                                                  '# Page X\n'))

        self.__update_test_layout(sitemap)
        self.__assert_stale(set(['source_b.test', 'page_x.markdown']))

        page_x = self.tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))

        self.tree.persist()

        # We now move the definition of symbol_3 to source_a.test,
        # we thus expect the generated page for source_a.test to be
        # stale because its source changed, same for source_b.test,
        # and page_x.markdown should be stale as well because the
        # definition of symbol_3 may have changed. The final
        # symbol layout should not have changed however.
        self.__create_src_file('source_a.test',
                               ['symbol_1', 'symbol_2', 'symbol_3'])
        self.__create_src_file('source_b.test', ['symbol_4'])
        self.__update_test_layout(sitemap)

        self.__assert_stale(
            set(['source_a.test', 'source_b.test', 'page_x.markdown']))

        page_x = self.tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))

        source_a_page = self.tree.get_pages()['source_a.test']
        self.assertEqual(source_a_page.symbol_names,
                         OrderedSet(['symbol_1', 'symbol_2']))

        self.tree.persist()

        # And we rollback again
        self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2'])
        self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4'])
        self.__update_test_layout(sitemap)

        self.__assert_stale(
            set(['source_a.test', 'source_b.test', 'page_x.markdown']))

        page_x = self.tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))

        source_a_page = self.tree.get_pages()['source_a.test']
        self.assertEqual(source_a_page.symbol_names,
                         OrderedSet(['symbol_1', 'symbol_2']))

        self.tree.persist()

        # Now we'll try removing page_x altogether
        self.__remove_md_file('page_x.markdown')
        inp = (u'index.markdown\n'
               '\ttest-index\n'
               '\t\ttest-section.markdown\n'
               '\t\t\tsource_a.test\n'
               '\t\tpage_y.markdown\n'
               '\tcore_page.markdown\n')

        new_sitemap = self.__parse_sitemap(inp)
        self.__update_test_layout(new_sitemap)
        self.__assert_stale(set(['source_b.test']))
        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names,
                         OrderedSet(['symbol_4', 'symbol_3']))
        self.tree.persist()

        # And rollback again
        self.__create_md_file('page_x.markdown', (u'---\n'
                                                  'symbols: [symbol_3]\n'
                                                  '...\n'
                                                  '# Page X\n'))
        self.__update_test_layout(sitemap)
        self.__assert_stale(set(['page_x.markdown', 'source_b.test']))

        page_x = self.tree.get_pages()['page_x.markdown']
        self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))

        source_b_page = self.tree.get_pages()['source_b.test']
        self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))

        self.tree.persist()

    def test_index_override_incremental(self):
        sitemap = self.__create_test_layout()
        self.tree.persist()
        index_page = self.tree.get_pages()['test-index']
        self.assertIn('source_b.test', index_page.subpages)

        self.incremental = True

        self.__touch_src_file('test-index.markdown')
        self.__update_test_layout(sitemap)
        index_page = self.tree.get_pages()['test-index']
        self.assertIn('source_b.test', index_page.subpages)
        self.tree.persist()

    def test_extension_index_only(self):
        inp = (u'test-index\n' '\ttest-section.markdown\n')
        self.test_ext.setup()
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file('test-section.markdown', u'# My test section\n')
        self.tree.parse_sitemap(sitemap)
        self.__assert_extension_names(
            self.tree, {
                u'test-index': 'test-extension',
                u'test-section.markdown': 'test-extension'
            })

    def test_extension_auto_sorted_override(self):
        self.__create_md_file('source_b.test.markdown',
                              (u'---\nauto-sort: true\n...\n# My override\n'))
        sitemap = (u'index.markdown\n'
                   '\ttest-index\n'
                   '\t\ttest-section.markdown\n'
                   '\t\t\tsource_b.test\n'
                   '\t\t\tsource_a.test\n'
                   '\t\tpage_x.markdown\n'
                   '\t\tpage_y.markdown\n'
                   '\tcore_page.markdown\n')
        _ = self.__create_test_layout(sitemap=sitemap)
        pages = self.tree.get_pages()
        self.assertTrue(pages['source_a.test'].pre_sorted)
        self.assertFalse(pages['source_b.test'].pre_sorted)

        self.__create_md_file('source_b.test.markdown', (u'# My override\n'))
        sitemap = (u'index.markdown\n'
                   '\ttest-index\n'
                   '\t\ttest-section.markdown\n'
                   '\t\t\tsource_b.test\n'
                   '\t\t\tsource_a.test\n'
                   '\t\tpage_x.markdown\n'
                   '\t\tpage_y.markdown\n'
                   '\tcore_page.markdown\n')
        _ = self.__create_test_layout(sitemap=sitemap)
        pages = self.tree.get_pages()
        self.assertTrue(pages['source_a.test'].pre_sorted)
        self.assertTrue(pages['source_b.test'].pre_sorted)

    def test_extension_implicit_override(self):
        self.__create_md_file(
            'source_b.test.markdown',
            (u'---\nsymbols:\n  - symbol_2\n...\n# My override\n'))
        _ = self.__create_test_layout()

        source_b = self.tree.get_pages()['source_b.test']
        self.assertEqual(os.path.basename(source_b.source_file),
                         'source_b.test.markdown')
        self.assertEqual(source_b.symbol_names, ['symbol_2', 'symbol_4'])

        source_a = self.tree.get_pages()['source_a.test']
        self.assertEqual(source_a.symbol_names, ['symbol_1'])

        out, _ = cmark.ast_to_html(source_b.ast, None)

        self.assertEqual(out, u'<h1>My override</h1>\n')
Пример #16
0
class Page(object):
    "Banana banana"
    meta_schema = {
        Optional('title'): And(str, len),
        Optional('symbols'): Schema([And(str, len)]),
        Optional('short-description'): And(str, len),
        Optional('render-subpages'): bool,
        Optional('auto-sort'): bool,
        Optional('full-width'): bool
    }

    # pylint: disable=too-many-arguments
    def __init__(self,
                 source_file,
                 ast,
                 output_path,
                 project_name,
                 meta=None,
                 raw_contents=None):
        "Banana banana"
        assert source_file
        basename = os.path.basename(source_file)
        name = os.path.splitext(basename)[0]
        ref = os.path.join(output_path,
                           re.sub(r'\W+', '-',
                                  os.path.splitext(basename)[0]))
        pagename = '%s.html' % ref

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.pre_sorted = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.typed_symbols = {}
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None
        self.build_path = None
        self.project_name = project_name
        self.cached_paths = OrderedSet()

        meta = meta or {}

        try:
            self.meta = Schema(Page.meta_schema).validate(meta)
        except SchemaError as _:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
            self.meta = meta

        self.symbol_names = OrderedSet(meta.get('symbols') or [])
        self.short_description = meta.get('short-description')
        self.render_subpages = meta.get('render-subpages', True)

        self.title = None
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, ref)

    def __getstate__(self):
        return {
            'ast': None,
            'build_path': None,
            'title': self.title,
            'raw_contents': self.raw_contents,
            'short_description': self.short_description,
            'extension_name': self.extension_name,
            'link': self.link,
            'meta': self.meta,
            'source_file': self.source_file,
            'comment': self.comment,
            'generated': self.generated,
            'is_stale': False,
            'formatted_contents': None,
            'detailed_description': None,
            'output_attrs': None,
            'symbols': [],
            'typed_symbols': {},
            'subpages': self.subpages,
            'symbol_names': self.symbol_names,
            'project_name': self.project_name,
            'pre_sorted': self.pre_sorted,
            'cached_paths': self.cached_paths,
            'render_subpages': self.render_subpages
        }

    def resolve_symbols(self, tree, database, link_resolver):
        """
        When this method is called, the page's symbol names are queried
        from `database`, and added to lists of actual symbols, sorted
        by symbol class.
        """
        typed_symbols_list = namedtuple('TypedSymbolsList',
                                        ['name', 'symbols'])

        for subclass in all_subclasses(Symbol):
            self.typed_symbols[subclass] = typed_symbols_list(
                subclass.get_plural_name(), [])

        all_syms = OrderedSet()
        for sym_name in self.symbol_names:
            sym = database.get_symbol(sym_name)
            self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
                                       database)

        if tree.project.is_toplevel:
            page_path = self.link.ref
        else:
            page_path = self.project_name + '/' + self.link.ref

        for sym in all_syms:
            sym.update_children_comments()
            self.__resolve_symbol(sym, link_resolver, page_path)
            self.symbol_names.add(sym.unique_name)

        for sym_type in [
                ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
        ]:
            syms = self.typed_symbols[sym_type].symbols

            if not syms:
                continue

            if self.title is None:
                self.title = syms[0].display_name
            if self.comment is None:
                self.comment = Comment(name=self.source_file)
                self.comment.short_description = syms[
                    0].comment.short_description
                self.comment.title = syms[0].comment.title
            break

    # pylint: disable=no-self-use
    def __fetch_comment(self, sym, database):
        old_comment = sym.comment
        new_comment = database.get_comment(sym.unique_name)
        sym.comment = Comment(sym.unique_name)

        if new_comment:
            sym.comment = new_comment
        elif old_comment:
            if old_comment.filename not in (ChangeTracker.all_stale_files |
                                            ChangeTracker.all_unlisted_files):
                sym.comment = old_comment

    def __format_page_comment(self, formatter, link_resolver):
        if not self.comment:
            return

        if self.comment.short_description:
            self.short_description = formatter.format_comment(
                self.comment.short_description, link_resolver).strip()
            if self.short_description.startswith('<p>'):
                self.short_description = self.short_description[3:-4]
        if self.comment.title:
            self.title = formatter.format_comment(self.comment.title,
                                                  link_resolver).strip()
            if self.title.startswith('<p>'):
                self.title = self.title[3:-4]

        if self.title:
            self.formatted_contents += '<h1>%s</h1>' % self.title

        self.formatted_contents += formatter.format_comment(
            self.comment, link_resolver)

    def format(self, formatter, link_resolver, output):
        """
        Banana banana
        """

        if not self.title and self.source_file:
            title = os.path.splitext(self.source_file)[0]
            self.title = os.path.basename(title).replace('-', ' ')

        self.formatted_contents = u''

        self.build_path = os.path.join(formatter.get_output_folder(self),
                                       self.link.ref)

        if self.ast:
            out, diags = cmark.ast_to_html(self.ast, link_resolver)
            for diag in diags:
                warn(diag.code,
                     message=diag.message,
                     filename=self.source_file)

            self.formatted_contents += out

        if not self.formatted_contents:
            self.__format_page_comment(formatter, link_resolver)

        self.output_attrs = defaultdict(lambda: defaultdict(dict))
        formatter.prepare_page_attributes(self)
        self.__format_symbols(formatter, link_resolver)
        self.detailed_description =\
            formatter.format_page(self)[0]

        if output:
            formatter.cache_page(self)

    # pylint: disable=no-self-use
    def get_title(self):
        """
        Banana banana
        """
        return self.title or 'unnamed'

    def __discover_title(self, meta):
        if meta is not None and 'title' in meta:
            self.title = meta['title']
        elif self.ast:
            self.title = cmark.title_from_ast(self.ast)

    def __format_symbols(self, formatter, link_resolver):
        for symbol in self.symbols:
            if symbol is None:
                continue
            debug(
                'Formatting symbol %s in page %s' %
                (symbol.unique_name, self.source_file), 'formatting')
            symbol.skip = not formatter.format_symbol(symbol, link_resolver)

    def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
                              database):
        if sym:
            self.__fetch_comment(sym, database)
            new_symbols = sum(tree.resolving_symbol_signal(self, sym), [])
            all_syms.add(sym)

            for symbol in new_symbols:
                self.__query_extra_symbols(symbol, all_syms, tree,
                                           link_resolver, database)

    def __resolve_symbol(self, symbol, link_resolver, page_path):
        symbol.resolve_links(link_resolver)

        symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)

        for link in symbol.get_extra_links():
            link.ref = "%s#%s" % (page_path, link.id_)

        tsl = self.typed_symbols.get(type(symbol))
        if tsl:
            tsl.symbols.append(symbol)
        self.symbols.append(symbol)

        debug(
            'Resolved symbol %s to page %s' %
            (symbol.display_name, self.link.ref), 'resolution')
Пример #17
0
class Page(object):
    "Banana banana"
    meta_schema = {Optional('title'): And(unicode, len),
                   Optional('symbols'): Schema([And(unicode, len)]),
                   Optional('short-description'): And(unicode, len)}

    resolving_symbol_signal = Signal()
    formatting_signal = Signal()

    def __init__(self, source_file, ast, meta=None, raw_contents=None):
        "Banana banana"
        assert source_file
        if os.path.isabs(source_file):
            basename = os.path.basename(source_file)
        else:
            basename = source_file.replace('/', '-')
        name = os.path.splitext(basename)[0]
        pagename = '%s.html' % name

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.typed_symbols = {}
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None

        meta = meta or {}

        try:
            self.meta = Schema(Page.meta_schema).validate(meta)
        except SchemaError as _:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (self.source_file,
                                                 str(_)))
            self.meta = meta

        self.symbol_names = OrderedSet(meta.get('symbols') or [])
        self.short_description = meta.get('short-description')

        self.title = None
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, name)

    def __getstate__(self):
        return {'ast': None,
                'title': self.title,
                'raw_contents': self.raw_contents,
                'short_description': self.short_description,
                'extension_name': self.extension_name,
                'link': self.link,
                'meta': self.meta,
                'source_file': self.source_file,
                'comment': self.comment,
                'generated': self.generated,
                'is_stale': False,
                'formatted_contents': None,
                'detailed_description': None,
                'output_attrs': None,
                'symbols': [],
                'typed_symbols': {},
                'subpages': self.subpages,
                'symbol_names': self.symbol_names}

    def resolve_symbols(self, doc_database, link_resolver):
        """
        When this method is called, the page's symbol names are queried
        from `doc_database`, and added to lists of actual symbols, sorted
        by symbol class.
        """
        typed_symbols_list = namedtuple(
            'TypedSymbolsList', ['name', 'symbols'])
        self.typed_symbols[Symbol] = typed_symbols_list('FIXME symbols', [])
        self.typed_symbols[FunctionSymbol] = typed_symbols_list(
            "Functions", [])
        self.typed_symbols[CallbackSymbol] = typed_symbols_list(
            "Callback Functions", [])
        self.typed_symbols[FunctionMacroSymbol] = typed_symbols_list(
            "Function Macros", [])
        self.typed_symbols[ConstantSymbol] = typed_symbols_list(
            "Constants", [])
        self.typed_symbols[ExportedVariableSymbol] = typed_symbols_list(
            "Exported Variables", [])
        self.typed_symbols[StructSymbol] = typed_symbols_list(
            "Data Structures", [])
        self.typed_symbols[EnumSymbol] = typed_symbols_list("Enumerations", [])
        self.typed_symbols[AliasSymbol] = typed_symbols_list("Aliases", [])
        self.typed_symbols[SignalSymbol] = typed_symbols_list("Signals", [])
        self.typed_symbols[PropertySymbol] = typed_symbols_list(
            "Properties", [])
        self.typed_symbols[VFunctionSymbol] = typed_symbols_list(
            "Virtual Methods", [])
        self.typed_symbols[ClassSymbol] = typed_symbols_list("Classes", [])
        self.typed_symbols[InterfaceSymbol] = typed_symbols_list("Interfaces", [])

        all_syms = OrderedSet()
        for sym_name in self.symbol_names:
            sym = doc_database.get_symbol(sym_name)
            self.__query_extra_symbols(sym, all_syms, link_resolver, doc_database)

        for sym in all_syms:
            sym.update_children_comments()
            self.__resolve_symbol(sym, link_resolver)
            self.symbol_names.add(sym.unique_name)

        class_syms = self.typed_symbols[ClassSymbol].symbols
        interface_syms = self.typed_symbols[InterfaceSymbol].symbols
        struct_syms = self.typed_symbols[StructSymbol].symbols

        if self.title is None:
            if class_syms:
                self.title = class_syms[0].display_name
            elif interface_syms:
                self.title = interface_syms[0].display_name
            elif struct_syms:
                self.title = struct_syms[0].display_name

        if self.comment is None:
            if class_syms and class_syms[0].comment:
                self.comment = class_syms[0].comment
            elif interface_syms and interface_syms[0].comment:
                self.comment = interface_syms[0].comment
            elif struct_syms and struct_syms[0].comment:
                self.comment = struct_syms[0].comment

    # pylint: disable=no-self-use
    def __fetch_comment(self, sym, doc_database):
        old_comment = sym.comment
        new_comment = doc_database.get_comment(sym.unique_name)
        sym.comment = Comment(sym.unique_name)

        if new_comment:
            sym.comment = new_comment
        elif old_comment:
            if not old_comment.filename in (ChangeTracker.all_stale_files |
                                            ChangeTracker.all_unlisted_files):
                sym.comment = old_comment

    def __format_page_comment(self, formatter, link_resolver):
        if not self.comment:
            return

        if self.comment.short_description:
            self.short_description = formatter.format_comment(
                self.comment.short_description, link_resolver).strip()
            if self.short_description.startswith('<p>'):
                self.short_description = self.short_description[3:-4]
        if self.comment.title:
            self.title = formatter.format_comment(
                self.comment.title, link_resolver).strip()
            if self.title.startswith('<p>'):
                self.title = self.title[3:-4]

        if self.title:
            self.formatted_contents += '<h1>%s</h1>' % self.title

        self.formatted_contents += formatter.format_comment(
            self.comment, link_resolver)

    def format(self, formatter, link_resolver, output):
        """
        Banana banana
        """

        if not self.title and self.source_file:
            title = os.path.splitext(self.source_file)[0]
            self.title = os.path.basename(title).replace('-', ' ')

        self.formatted_contents = u''

        if self.ast:
            out, diags = cmark.ast_to_html(self.ast, link_resolver)
            for diag in diags:
                warn(
                    diag.code,
                    message=diag.message,
                    filename=self.source_file)

            self.formatted_contents += out
        else:
            self.__format_page_comment(formatter, link_resolver)

        self.output_attrs = defaultdict(lambda: defaultdict(dict))
        formatter.prepare_page_attributes(self)
        Page.formatting_signal(self, formatter)
        self.__format_symbols(formatter, link_resolver)
        self.detailed_description =\
            formatter.format_page(self)[0]

        if output:
            formatter.write_page(self, output)

    # pylint: disable=no-self-use
    def get_title(self):
        """
        Banana banana
        """
        return self.title or 'unnamed'

    def __discover_title(self, meta):
        if meta is not None and 'title' in meta:
            self.title = meta['title']
        elif self.ast:
            self.title = cmark.title_from_ast(self.ast)

    def __format_symbols(self, formatter, link_resolver):
        for symbol in self.symbols:
            if symbol is None:
                continue
            debug('Formatting symbol %s in page %s' % (
                symbol.unique_name, self.source_file), 'formatting')
            symbol.skip = not formatter.format_symbol(symbol, link_resolver)

    def __query_extra_symbols(self, sym, all_syms, link_resolver, doc_database):
        if sym:
            self.__fetch_comment(sym, doc_database)
            new_symbols = sum(Page.resolving_symbol_signal(self, sym),
                              [])
            all_syms.add(sym)

            for symbol in new_symbols:
                self.__query_extra_symbols(symbol, all_syms, link_resolver, doc_database)

    def __resolve_symbol(self, symbol, link_resolver):
        symbol.resolve_links(link_resolver)

        symbol.link.ref = "%s#%s" % (self.link.ref, symbol.unique_name)

        for link in symbol.get_extra_links():
            link.ref = "%s#%s" % (self.link.ref, link.id_)

        tsl = self.typed_symbols[type(symbol)]
        tsl.symbols.append(symbol)
        self.symbols.append(symbol)

        debug('Resolved symbol %s to page %s' %
              (symbol.display_name, self.link.ref), 'resolution')