Пример #1
0
    def parse_config(self, config, toplevel=False):
        """Parses @config setting up @self state."""
        self.sitemap_path = config.get_path('sitemap')

        if self.sitemap_path is None:
            error('invalid-config', 'No sitemap was provided')

        self.include_paths = OrderedSet([])

        index_file = config.get_index()
        if index_file:
            if not os.path.exists(index_file):
                error('invalid-config',
                      'The provided index "%s" does not exist' % index_file)
            self.include_paths |= OrderedSet([os.path.dirname(index_file)])

        self.include_paths |= OrderedSet(config.get_paths('include_paths'))

        self.is_toplevel = toplevel

        self.tree = Tree(self, self.app)

        self.__create_extensions()

        for extension in list(self.extensions.values()):
            if toplevel:
                extension.parse_toplevel_config(config)
            extension.parse_config(config)

        if not toplevel and config.conf_file:
            self.app.change_tracker.add_hard_dependency(config.conf_file)

        self.extra_asset_folders = OrderedSet(config.get_paths('extra_assets'))
Пример #2
0
    def __update_meta(self, meta):
        for key, value in meta.items():
            try:
                self.meta.update(
                    Schema(Page.meta_schema).validate(
                        {key.replace('_', '-').lower(): value}))
            except SchemaError as _:
                warn(
                    'invalid-page-metadata',
                    '%s: Invalid metadata: \n%s, discarding metadata' %
                    (self.source_file, str(_)))

        if not self.meta.get('extra'):
            self.meta['extra'] = defaultdict()

        self.title = meta.get('title', self.title)
        self.thumbnail = meta.get('thumbnail')
        self.listed_symbols = OrderedSet(
            meta.get('symbols') or self.symbol_names)
        self.private_symbols = OrderedSet(
            meta.get('private-symbols') or self.private_symbols)
        self.symbol_names = OrderedSet(
            meta.get('symbols') or self.symbol_names)
        self.short_description = meta.get('short-description',
                                          self.short_description)
        self.render_subpages = meta.get('render-subpages',
                                        self.render_subpages)
Пример #3
0
 def prepare_page_attributes(self, page):
     """
     Banana banana
     """
     page.output_attrs['html']['scripts'] = OrderedSet()
     page.output_attrs['html']['stylesheets'] = OrderedSet()
     page.output_attrs['html']['extra_html'] = []
     page.output_attrs['html']['extra_footer_html'] = []
     if HtmlFormatter.add_anchors:
         page.output_attrs['html']['scripts'].add(
             os.path.join(HERE, 'html_assets', 'css.escape.js'))
     Formatter.prepare_page_attributes(self, page)
Пример #4
0
    def __resolve_patterns(self, source_patterns, from_conf):
        if source_patterns is None:
            return OrderedSet()

        all_files = OrderedSet()
        for item in source_patterns:
            item = self.__abspath(item, from_conf)
            if '*' in item:
                all_files |= glob.glob(item)
            else:
                all_files.add(item)

        return all_files
Пример #5
0
 def prepare_page_attributes(self, page):
     """
     Banana banana
     """
     self._current_page = page
     page.output_attrs['html']['scripts'] = OrderedSet()
     page.output_attrs['html']['stylesheets'] = OrderedSet()
     page.output_attrs['html']['extra_html'] = []
     page.output_attrs['html']['edit_button'] = ''
     page.output_attrs['html']['extra_footer_html'] = []
     if Formatter.add_anchors:
         page.output_attrs['html']['scripts'].add(
             os.path.join(HERE, 'assets', 'css.escape.js'))
Пример #6
0
    def __init__(self,
                 source_file,
                 ast,
                 output_path,
                 project_name,
                 meta=None,
                 raw_contents=None):
        "Banana banana"
        assert source_file
        basename = os.path.basename(source_file)
        name = os.path.splitext(basename)[0]
        ref = os.path.join(output_path,
                           re.sub(r'\W+', '-',
                                  os.path.splitext(basename)[0]))
        pagename = '%s.html' % ref

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.pre_sorted = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.typed_symbols = {}
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None
        self.build_path = None
        self.project_name = project_name
        self.cached_paths = OrderedSet()

        meta = meta or {}

        try:
            self.meta = Schema(Page.meta_schema).validate(meta)
        except SchemaError as _:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
            self.meta = meta

        self.symbol_names = OrderedSet(meta.get('symbols') or [])
        self.short_description = meta.get('short-description')
        self.render_subpages = meta.get('render-subpages', True)

        self.title = None
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, ref)
Пример #7
0
    def setUp(self):
        here = os.path.dirname(__file__)
        self.__md_dir = os.path.abspath(
            os.path.join(here, 'tmp-markdown-files'))
        self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private'))
        self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
        self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
        self.__remove_tmp_dirs()
        os.mkdir(self.__md_dir)
        os.mkdir(self.__priv_dir)
        os.mkdir(self.__src_dir)
        os.mkdir(self.get_generated_doc_folder())
        self.include_paths = OrderedSet([self.__md_dir])
        self.include_paths.add(self.get_generated_doc_folder())

        # Using the real doc database is too costly, tests should be lightning
        # fast (and they are)
        self.doc_database = DocDatabase()
        self.doc_database.setup(self.__priv_dir)
        self.link_resolver = LinkResolver(self.doc_database)

        self.change_tracker = ChangeTracker()

        self.sitemap_parser = SitemapParser()

        self.test_ext = TestExtension(self)
        self.core_ext = CoreExtension(self)
Пример #8
0
    def get_stale_files(self, all_files, fileset_name):
        """
        Banana banana
        """
        stale = OrderedSet()

        previous_mtimes = self.mtimes[fileset_name]
        new_mtimes = defaultdict()

        for filename in all_files:
            mtime = get_mtime(filename)
            prev_mtime = previous_mtimes.pop(filename, None)
            new_mtimes[filename] = mtime
            if mtime == prev_mtime:
                continue

            stale.add(filename)

        self.mtimes[fileset_name] = new_mtimes

        unlisted = set(previous_mtimes.keys())

        ChangeTracker.all_stale_files |= stale
        ChangeTracker.all_unlisted_files |= unlisted

        return stale, unlisted
Пример #9
0
 def get_pagename(self, name):
     self.__find_package_root()
     for path in OrderedSet([self.__package_root]) | self.source_roots:
         commonprefix = os.path.commonprefix([path, name])
         if commonprefix == path:
             return os.path.relpath(name, path)
     return name
Пример #10
0
    def __init__(self, app, dependency_map=None, page_map=None):
        self.app = app
        self.tree = None
        self.include_paths = None
        self.extensions = OrderedDict()
        self.tag_validators = {}
        self.project_name = None
        self.project_version = None
        self.sanitized_name = None
        self.sitemap_path = None
        self.subprojects = {}
        self.extra_asset_folders = OrderedSet()
        self.extra_assets = {}

        if dependency_map is None:
            self.dependency_map = {}
        else:
            self.dependency_map = dependency_map

        if page_map is None:
            self.page_map = {}
        else:
            self.page_map = page_map

        if os.name == 'nt':
            self.datadir = os.path.join(os.path.dirname(__file__), '..',
                                        'share')
        else:
            self.datadir = "/usr/share"

        self.formatted_signal = Signal()
        self.written_out_signal = Signal()

        self.is_toplevel = False
Пример #11
0
    def parse_config(self, config):
        """
        Override this, making sure to chain up first, if your extension adds
        its own custom command line arguments, or you want to do any further
        processing on the automatically added arguments.

        The default implementation will set attributes on the extension:
        - 'sources': a set of absolute paths to source files for this extension
        - 'index': absolute path to the index for this extension

        Additionally, it will set an attribute for each argument added with
        `Extension.add_path_argument` or `Extension.add_paths_argument`, with
        the extension's `Extension.argument_prefix` stripped, and dashes
        changed to underscores.

        Args:
            config: a `config.Config` instance
        """
        prefix = self.argument_prefix
        self.sources = config.get_sources(prefix)
        self.smart_sources = [
            self._get_smart_filename(s) for s in self.sources]
        self.index = config.get_index(prefix)
        self.source_roots = OrderedSet(
            config.get_paths('%s_source_roots' % prefix))

        for arg, dest in list(self.paths_arguments.items()):
            val = config.get_paths(arg)
            setattr(self, dest, val)

        for arg, dest in list(self.path_arguments.items()):
            val = config.get_path(arg)
            setattr(self, dest, val)

        self.formatter.parse_config(config)
Пример #12
0
    def _resolve_placeholder(self, tree, name, include_paths):
        self.__find_package_root()

        if name == '%s-index' % self.argument_prefix:
            if self.index:
                path = find_file(self.index, include_paths)
                if path is None:
                    self.error("invalid-config",
                               "Could not find index file %s" % self.index)
                return PageResolutionResult(True, path, None,
                                            self.extension_name)
            return PageResolutionResult(True, None, None, self.extension_name)

        if self.smart_index:
            for path in OrderedSet([self.__package_root]) | self.source_roots:
                possible_path = os.path.join(path, name)
                if possible_path in self._get_all_sources():
                    override_path = find_file('%s.markdown' % name,
                                              include_paths)

                    if override_path:
                        return PageResolutionResult(True, override_path, None,
                                                    None)

                    return PageResolutionResult(
                        True, None, self.__get_rel_source_path(possible_path),
                        None)

        return None
Пример #13
0
 def test_smart_key_default(self):
     sources = []
     sources.append(self._create_src_file('source_a.test', ['symbol_1']))
     self.test_ext.sources = sources
     self.test_ext.setup()
     self.assertDictEqual(self.test_ext.get_created_symbols(),
                          {sources[0]: OrderedSet(['symbol_1'])})
Пример #14
0
    def get_dependencies(self):
        """
        Retrieve the set of all dependencies for a given configuration.

        Returns:
            utils.utils.OrderedSet: The set of all dependencies for the
                tracked configuration.
        """
        all_deps = OrderedSet()
        for key, _ in list(self.__config.items()):
            if key in self.__cli:
                continue

            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1 - 1])

        for key, _ in list(self.__cli.items()):
            if key.endswith('sources'):
                all_deps |= self.get_sources(key[:len('sources') * -1 - 1])

        if self.conf_file is not None:
            all_deps.add(self.conf_file)

        all_deps.add(self.get_path("sitemap", rel_to_cwd=True))

        cwd = os.getcwd()
        return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
Пример #15
0
    def test_parse_yaml(self):
        inp = (u'index.markdown\n')
        sitemap = self.__parse_sitemap(inp)
        self.__create_md_file(
            'index.markdown',
            (u'---\n'
             'title: A random title\n'
             'symbols: [symbol_1, symbol_2]\n'
             '...\n'
             '# My documentation\n'))

        self.tree.parse_sitemap(sitemap)

        pages = self.tree.get_pages()
        page = pages.get('index.markdown')

        out, _ = cmark.ast_to_html(page.ast, None)

        self.assertEqual(
            out,
            u'<h1>My documentation</h1>\n')

        self.assertEqual(page.title, u'A random title')

        self.assertEqual(
            page.symbol_names,
            OrderedSet(['symbol_1',
                        'symbol_2']))
Пример #16
0
    def __find_structure_pagename(self, node, unique_name, is_class):
        filename = self.__get_symbol_filename(unique_name, node)
        if filename != self.__default_page:
            return filename

        if self.__symbol_is_relocated(unique_name, None):
            return self.__default_page

        if not is_class:
            sym = self.__class_gtype_structs.get(node.attrib['name'])
            if sym and sym.filename:
                return sym.filename

        filenames = []
        for cnode in node:
            cunique_name = get_symbol_names(cnode)[0]
            if not cunique_name:
                continue
            fname = self.__get_symbol_filename(cunique_name, cnode)
            if fname != self.__default_page:
                if cnode.tag == core_ns('constructor'):
                    filenames.insert(0, fname)
                else:
                    filenames.append(fname)

        unique_filenames = list(OrderedSet(filenames))
        if not filenames:
            # Did not find any symbols, trying to can get information
            # about the class structure linked to that object class.
            nextnode = node.getnext()
            name = node.attrib['name']
            if nextnode is not None and nextnode.tag == core_ns('record'):
                nextnode_classfor = nextnode.attrib.get(
                    glib_ns('is-gtype-struct-for'))
                if nextnode_classfor == name:
                    nunique_name = get_symbol_names(nextnode)[0]
                    filename = self.__get_symbol_filename(nunique_name)

            if filename == self.__default_page:
                self.warn(
                    "no-location-indication",
                    "No way to determine where %s should land"
                    " putting it to %s."
                    " Document the symbol for smart indexing to work" %
                    (unique_name, os.path.basename(filename)))
        else:
            filename = unique_filenames[0]
            if len(unique_filenames) > 1:
                self.warn(
                    "no-location-indication",
                    " Going wild here to determine where %s needs to land"
                    " as we could detect the following possibilities: %s." %
                    (unique_name, unique_filenames))
            else:
                self.debug(
                    " No class comment for %s determined that it should"
                    " land into %s with all other class related documentation."
                    % (unique_name, os.path.basename(filename)))

        return filename
Пример #17
0
    def parse_config(self, config):
        super(GIExtension, self).parse_config(config)
        ALL_GIRS.update({os.path.basename(s): s for s in self.sources})
        self.c_sources = config.get_sources('gi-c')
        self.source_roots = OrderedSet(config.get_paths('gi_c_source_roots'))

        chosen_languages = [l.lower() for l in config.get('languages', [])]
        languages = []
        for lang_type in get_language_classes():
            if lang_type.language_name in chosen_languages:
                languages.append(lang_type())

        if languages:
            self.languages = languages
        else:
            self.languages = [
                lang_type() for lang_type in get_language_classes()
            ]

        # Make sure C always gets formatted first
        c_language = self.get_language('c')
        if c_language:
            self.languages.remove(c_language)
            self.languages.insert(0, c_language)

        for gir_file in self.sources:
            gir_root = etree.parse(gir_file).getroot()
            cache_nodes(gir_root, ALL_GIRS, self.languages)
Пример #18
0
    def __get_user_symbols(self, tree, index):
        symbols = self.__get_listed_symbols_in_markdown(tree, index)
        private_symbols = set()
        parented_symbols = defaultdict(list)

        for source_file, symbols_names in list(self._created_symbols.items()):
            if source_file.endswith(('.markdown', '.md')):
                continue

            for symname in symbols_names:
                symbol = self.app.database.get_symbol(symname)
                if not symbol.parent_name:
                    continue

                if symbol.parent_name in symbols:
                    symbols[symbol.unique_name] = symbols[symbol.parent_name]
                else:
                    parented_symbols[symbol.parent_name].append(symname)

            page_name = self.__get_page(tree, source_file)[1]
            comment = self.__get_comment_for_page(source_file, page_name)
            if not comment:
                continue

            for symname in comment.meta.get("private-symbols", OrderedSet()):
                private_symbols.add(symname)

            comment_syms, located_parented_symbols = self.__list_symbols_in_comment(
                comment, parented_symbols, source_file, page_name)
            if comment_syms:
                comment.meta['symbols'].extend(located_parented_symbols)
                symbols.update(comment_syms)

        return set(symbols.keys()), set(symbols.values()), private_symbols
Пример #19
0
    def __init__(self,
                 source_file,
                 ast,
                 output_path,
                 project_name,
                 meta=None,
                 raw_contents=None):
        "Banana banana"
        assert source_file
        basename = os.path.basename(source_file)
        name = os.path.splitext(basename)[0]
        ref = os.path.join(output_path,
                           re.sub(r'\W+', '-',
                                  os.path.splitext(basename)[0]))
        pagename = '%s.html' % ref

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.pre_sorted = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.private_symbols = []
        self.typed_symbols = OrderedDict()
        self.by_parent_symbols = OrderedDict()
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None
        self.build_path = None
        self.project_name = project_name
        self.cached_paths = OrderedSet()

        meta = meta or {}
        self.listed_symbols = []
        self.symbol_names = []
        self.short_description = None
        self.render_subpages = True
        self.title = ''
        self.meta = Schema(Page.meta_schema).validate({})
        self.__update_meta(meta)
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, ref)
Пример #20
0
    def __resolve_patterns(self, source_patterns, from_conf):
        if source_patterns is None:
            return OrderedSet()

        all_files = OrderedSet()
        for item in source_patterns:
            item = self.__abspath(item, from_conf)

            if item in all_files:
                continue

            if not os.path.exists(item):
                all_files |= glob.glob(item)
            else:
                all_files.add(item)

        return all_files
Пример #21
0
    def get_possible_path(self, name):
        self.__find_package_root()

        for path in OrderedSet([self.__package_root]) | self.source_roots:
            possible_path = os.path.join(path, name)
            if possible_path in self._get_all_sources():
                return self._get_smart_filename(possible_path)
        return None
Пример #22
0
 def test_smart_key_custom(self):
     sources = []
     sources.append(
         self._create_src_file('source_a.test', ['custom_key', 'symbol_1']))
     self.test_ext.sources = sources
     self.test_ext.use_custom_key = True
     self.test_ext.setup()
     self.assertDictEqual(self.test_ext.get_created_symbols(),
                          {'custom_key': OrderedSet(['symbol_1'])})
Пример #23
0
    def __get_page(self, tree, source_file):
        page_name = self.__get_rel_source_path(source_file)
        for path in OrderedSet([self.__package_root]) | self.source_roots:
            possible_name = os.path.relpath(source_file, path)
            page = tree.get_pages().get(possible_name)
            if page:
                return page, page_name

        return page, page_name
Пример #24
0
    def __parse_config(self):
        """
        Banana banana
        """
        output = self.config.get_path('output') or None
        self.sitemap_path = self.config.get_path('sitemap')

        if self.sitemap_path is None:
            error('invalid-config', 'No sitemap was provided')

        if output is not None:
            self.output = os.path.abspath(output)
        else:
            self.output = None

        self.project_name = self.config.get('project_name', None)
        self.project_version = self.config.get('project_version', None)
        self.output_format = self.config.get('output_format')

        if self.output_format not in ["html"]:
            error('invalid-config',
                  'Unsupported output format : %s' % self.output_format)

        self.__index_file = self.config.get_index()
        if self.__index_file is None:
            error('invalid-config', 'index is required')
        if not os.path.exists(self.__index_file):
            error('invalid-config',
                  'The provided index "%s" does not exist' % self.__index_file)

        cmd_line_includes = self.config.get_paths('include_paths')
        self.__base_doc_folder = os.path.dirname(self.__index_file)
        self.include_paths = OrderedSet([self.__base_doc_folder])
        self.include_paths |= OrderedSet(cmd_line_includes)
        self.__create_change_tracker()
        self.__setup_private_folder()
        self.__setup_database()

        self.__create_extensions()

        if self.__conf_file:
            self.change_tracker.add_hard_dependency(self.__conf_file)
Пример #25
0
    def __init__(self, app, project):
        """Constructor for `Extension`.

        This should never get called directly.

        Args:
            project: The `project.Project` instance which documentation
                is being generated.
        """
        self.project = project
        self.app = app
        self.sources = set()
        self.smart_sources = []
        self.index = None
        self.source_roots = OrderedSet()
        self._created_symbols = DefaultOrderedDict(OrderedSet)
        self.__package_root = None
        self.__toplevel_comments = OrderedSet()

        self.formatter = self._make_formatter()
Пример #26
0
    def __init__(self, source_file, ast, meta=None, raw_contents=None):
        "Banana banana"
        assert source_file
        if os.path.isabs(source_file):
            basename = os.path.basename(source_file)
        else:
            basename = source_file.replace('/', '-')
        name = os.path.splitext(basename)[0]
        pagename = '%s.html' % name

        self.ast = ast
        self.extension_name = None
        self.source_file = source_file
        self.raw_contents = raw_contents
        self.comment = None
        self.generated = False
        self.output_attrs = None
        self.subpages = OrderedSet()
        self.symbols = []
        self.typed_symbols = {}
        self.is_stale = True
        self.formatted_contents = None
        self.detailed_description = None

        meta = meta or {}

        try:
            self.meta = Schema(Page.meta_schema).validate(meta)
        except SchemaError as _:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
            self.meta = meta

        self.symbol_names = OrderedSet(meta.get('symbols') or [])
        self.short_description = meta.get('short-description')

        self.title = None
        self.__discover_title(meta)
        self.link = Link(pagename, self.title or name, name)
Пример #27
0
    def get_pagename(self, name):
        self.__find_package_root()
        # Find the longest prefix
        longest = None
        for path in OrderedSet([self.__package_root]) | self.source_roots:
            commonprefix = os.path.commonprefix([path, name])
            if commonprefix == path and (longest is None or len(path) > len(longest)):
                longest = path

        if longest is not None:
            return os.path.relpath(name, longest)

        return name
Пример #28
0
        def setup_subpages(pagenames, get_pagename):
            """Setup subpages for pages with names in @pagenames"""
            sitemap_pages = sitemap.get_all_sources()
            for pagename in pagenames:
                page = self.__all_pages[get_pagename(pagename)]

                subpages = sitemap_pages.get(get_pagename(pagename), [])
                page.subpages = OrderedSet(subpages) | page.subpages
                for subpage_name in page.subpages:
                    if subpage_name not in unlisted_pagenames:
                        subpage = self.__all_pages[subpage_name]
                        if not subpage.meta.get('auto-sort', False):
                            subpage.pre_sorted = True
                page.subpages -= unlisted_pagenames
Пример #29
0
    def test_extension_override(self):
        self.__create_md_file('source_a.test.markdown', (u'# My override\n'))
        _ = self.__create_test_layout()
        page = self.tree.get_pages()['source_a.test']

        self.assertEqual(page.symbol_names,
                         OrderedSet(['symbol_1', 'symbol_2']))

        self.assertEqual(os.path.basename(page.source_file),
                         'source_a.test.markdown')

        out, _ = cmark.ast_to_html(page.ast, None)

        self.assertEqual(out, u'<h1>My override</h1>\n')
Пример #30
0
    def __resolve_patterns(self, source_patterns, from_conf):
        if source_patterns is None:
            return OrderedSet()

        cache_key = self.__get_key(source_patterns, from_conf)
        all_files = Config.__pattern_cache.get(cache_key, OrderedSet())
        if all_files:
            return all_files

        for item in source_patterns:
            item = self.__abspath(item, from_conf)

            if item in all_files:
                continue

            if not os.path.exists(item):
                all_files |= glob.glob(item)
            else:
                all_files.add(item)

        Config.__pattern_cache[cache_key] = all_files

        return all_files