def __update_meta(self, meta): for key, value in meta.items(): try: self.meta.update( Schema(Page.meta_schema).validate( {key.replace('_', '-').lower(): value})) except SchemaError as _: warn( 'invalid-page-metadata', '%s: Invalid metadata: \n%s, discarding metadata' % (self.source_file, str(_))) if not self.meta.get('extra'): self.meta['extra'] = defaultdict() self.title = meta.get('title', self.title) self.thumbnail = meta.get('thumbnail') self.listed_symbols = OrderedSet( meta.get('symbols') or self.symbol_names) self.private_symbols = OrderedSet( meta.get('private-symbols') or self.private_symbols) self.symbol_names = OrderedSet( meta.get('symbols') or self.symbol_names) self.short_description = meta.get('short-description', self.short_description) self.render_subpages = meta.get('render-subpages', self.render_subpages)
def get_stale_files(self, all_files, fileset_name): """ Banana banana """ stale = OrderedSet() previous_mtimes = self.mtimes[fileset_name] new_mtimes = defaultdict() for filename in all_files: mtime = get_mtime(filename) prev_mtime = previous_mtimes.pop(filename, None) new_mtimes[filename] = mtime if mtime == prev_mtime: continue stale.add(filename) self.mtimes[fileset_name] = new_mtimes unlisted = set(previous_mtimes.keys()) ChangeTracker.all_stale_files |= stale ChangeTracker.all_unlisted_files |= unlisted return stale, unlisted
def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath( os.path.join(here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self)
def get_dependencies(self): """ Retrieve the set of all dependencies for a given configuration. Returns: utils.utils.OrderedSet: The set of all dependencies for the tracked configuration. """ all_deps = OrderedSet() for key, _ in self.__config.items(): if key in self.__cli: continue if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1]) for key, _ in self.__cli.items(): if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1]) if self.__conf_file is not None: all_deps.add(self.__conf_file) all_deps.add(self.get_path("sitemap", rel_to_cwd=True)) cwd = os.getcwd() return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def parse_config(self, config, toplevel=False): """Parses @config setting up @self state.""" self.sitemap_path = config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') self.include_paths = OrderedSet([]) index_file = config.get_index() if index_file: if not os.path.exists(index_file): error('invalid-config', 'The provided index "%s" does not exist' % index_file) self.include_paths |= OrderedSet([os.path.dirname(index_file)]) self.include_paths |= OrderedSet(config.get_paths('include_paths')) self.is_toplevel = toplevel self.tree = Tree(self, self.app) self.__create_extensions() for extension in list(self.extensions.values()): if toplevel: extension.parse_toplevel_config(config) extension.parse_config(config) if not toplevel and config.conf_file: self.app.change_tracker.add_hard_dependency(config.conf_file) self.extra_asset_folders = OrderedSet(config.get_paths('extra_assets'))
def get_stale_files(self, all_files, fileset_name): """ Banana banana """ stale = OrderedSet() previous_mtimes = self.mtimes[fileset_name] new_mtimes = defaultdict() for filename in all_files: mtime = get_mtime(filename) prev_mtime = previous_mtimes.pop(filename, None) new_mtimes[filename] = mtime if mtime == prev_mtime: continue stale.add(filename) self.mtimes[fileset_name] = new_mtimes unlisted = set(previous_mtimes.keys()) ChangeTracker.all_stale_files |= stale ChangeTracker.all_unlisted_files |= unlisted return stale, unlisted
def prepare_page_attributes(self, page): """ Banana banana """ page.output_attrs['html']['scripts'] = OrderedSet() page.output_attrs['html']['stylesheets'] = OrderedSet() page.output_attrs['html']['extra_html'] = [] page.output_attrs['html']['extra_footer_html'] = [] if HtmlFormatter.add_anchors: page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'html_assets', 'css.escape.js')) Formatter.prepare_page_attributes(self, page)
def __resolve_patterns(self, source_patterns, from_conf): if source_patterns is None: return OrderedSet() all_files = OrderedSet() for item in source_patterns: item = self.__abspath(item, from_conf) if '*' in item: all_files |= glob.glob(item) else: all_files.add(item) return all_files
def prepare_page_attributes(self, page): """ Banana banana """ self._current_page = page page.output_attrs['html']['scripts'] = OrderedSet() page.output_attrs['html']['stylesheets'] = OrderedSet() page.output_attrs['html']['extra_html'] = [] page.output_attrs['html']['edit_button'] = '' page.output_attrs['html']['extra_footer_html'] = [] if Formatter.add_anchors: page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'assets', 'css.escape.js'))
def __init__(self, source_file, ast, output_path, project_name, meta=None, raw_contents=None): "Banana banana" assert source_file basename = os.path.basename(source_file) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.pre_sorted = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None self.build_path = None self.project_name = project_name self.cached_paths = OrderedSet() meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.render_subpages = meta.get('render-subpages', True) self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, ref)
def test_parse_yaml(self): inp = (u'index.markdown\n') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'---\n' 'title: A random title\n' 'symbols: [symbol_1, symbol_2]\n' '...\n' '# My documentation\n')) self.tree.parse_sitemap(sitemap) pages = self.tree.get_pages() page = pages.get('index.markdown') out, _ = cmark.ast_to_html(page.ast, None) self.assertEqual( out, u'<h1>My documentation</h1>\n') self.assertEqual(page.title, u'A random title') self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2']))
def __find_structure_pagename(self, node, unique_name, is_class): filename = self.__get_symbol_filename(unique_name, node) if filename != self.__default_page: return filename if self.__symbol_is_relocated(unique_name, None): return self.__default_page if not is_class: sym = self.__class_gtype_structs.get(node.attrib['name']) if sym and sym.filename: return sym.filename filenames = [] for cnode in node: cunique_name = get_symbol_names(cnode)[0] if not cunique_name: continue fname = self.__get_symbol_filename(cunique_name, cnode) if fname != self.__default_page: if cnode.tag == core_ns('constructor'): filenames.insert(0, fname) else: filenames.append(fname) unique_filenames = list(OrderedSet(filenames)) if not filenames: # Did not find any symbols, trying to can get information # about the class structure linked to that object class. nextnode = node.getnext() name = node.attrib['name'] if nextnode is not None and nextnode.tag == core_ns('record'): nextnode_classfor = nextnode.attrib.get( glib_ns('is-gtype-struct-for')) if nextnode_classfor == name: nunique_name = get_symbol_names(nextnode)[0] filename = self.__get_symbol_filename(nunique_name) if filename == self.__default_page: self.warn( "no-location-indication", "No way to determine where %s should land" " putting it to %s." " Document the symbol for smart indexing to work" % (unique_name, os.path.basename(filename))) else: filename = unique_filenames[0] if len(unique_filenames) > 1: self.warn( "no-location-indication", " Going wild here to determine where %s needs to land" " as we could detect the following possibilities: %s." % (unique_name, unique_filenames)) else: self.debug( " No class comment for %s determined that it should" " land into %s with all other class related documentation." % (unique_name, os.path.basename(filename))) return filename
def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath(os.path.join( here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join( here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join( here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join( here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self)
def parse_config(self, config): super(GIExtension, self).parse_config(config) ALL_GIRS.update({os.path.basename(s): s for s in self.sources}) self.c_sources = config.get_sources('gi-c') self.source_roots = OrderedSet(config.get_paths('gi_c_source_roots')) chosen_languages = [l.lower() for l in config.get('languages', [])] languages = [] for lang_type in get_language_classes(): if lang_type.language_name in chosen_languages: languages.append(lang_type()) if languages: self.languages = languages else: self.languages = [ lang_type() for lang_type in get_language_classes() ] # Make sure C always gets formatted first c_language = self.get_language('c') if c_language: self.languages.remove(c_language) self.languages.insert(0, c_language) for gir_file in self.sources: gir_root = etree.parse(gir_file).getroot() cache_nodes(gir_root, ALL_GIRS, self.languages)
def __init__(self, source_file, ast, meta=None, raw_contents=None): "Banana banana" assert source_file name = os.path.splitext(os.path.basename(source_file))[0] pagename = '%s.html' % name self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: error('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, name)
def get_pagename(self, name): self.__find_package_root() for path in OrderedSet([self.__package_root]) | self.source_roots: commonprefix = os.path.commonprefix([path, name]) if commonprefix == path: return os.path.relpath(name, path) return name
def __init__(self, app, project): super().__init__(app, project) self.cache = {} self.c_sources = [] self.cache_file = None self.plugin = None self.__elements = {} self.__raw_comment_parser = GtkDocParser(project, section_file_matching=False) self.__plugins = None self.__toplevel_comments = OrderedSet() self.list_plugins_page = None # If we have a plugin with only one element, we render it on the plugin # page. self.unique_feature = None self.__on_index_symbols = []
def __init__(self, app, dependency_map=None, page_map=None): self.app = app self.tree = None self.include_paths = None self.extensions = OrderedDict() self.tag_validators = {} self.project_name = None self.project_version = None self.sanitized_name = None self.sitemap_path = None self.subprojects = {} self.extra_asset_folders = OrderedSet() self.extra_assets = {} if dependency_map is None: self.dependency_map = {} else: self.dependency_map = dependency_map if page_map is None: self.page_map = {} else: self.page_map = page_map if os.name == 'nt': self.datadir = os.path.join(os.path.dirname(__file__), '..', 'share') else: self.datadir = "/usr/share" self.formatted_signal = Signal() self.written_out_signal = Signal() self.is_toplevel = False
def __get_user_symbols(self, tree, index): symbols = self.__get_listed_symbols_in_markdown(tree, index) private_symbols = set() parented_symbols = defaultdict(list) for source_file, symbols_names in list(self._created_symbols.items()): if source_file.endswith(('.markdown', '.md')): continue for symname in symbols_names: symbol = self.app.database.get_symbol(symname) if not symbol.parent_name: continue if symbol.parent_name in symbols: symbols[symbol.unique_name] = symbols[symbol.parent_name] else: parented_symbols[symbol.parent_name].append(symname) page_name = self.__get_page(tree, source_file)[1] comment = self.__get_comment_for_page(source_file, page_name) if not comment: continue for symname in comment.meta.get("private-symbols", OrderedSet()): private_symbols.add(symname) comment_syms, located_parented_symbols = self.__list_symbols_in_comment( comment, parented_symbols, source_file, page_name) if comment_syms: comment.meta['symbols'].extend(located_parented_symbols) symbols.update(comment_syms) return set(symbols.keys()), set(symbols.values()), private_symbols
def _resolve_placeholder(self, tree, name, include_paths): self.__find_package_root() if name == '%s-index' % self.argument_prefix: if self.index: path = find_file(self.index, include_paths) if path is None: self.error("invalid-config", "Could not find index file %s" % self.index) return PageResolutionResult(True, path, None, self.extension_name) return PageResolutionResult(True, None, None, self.extension_name) if self.smart_index: for path in OrderedSet([self.__package_root]) | self.source_roots: possible_path = os.path.join(path, name) if possible_path in self._get_all_sources(): override_path = find_file('%s.markdown' % name, include_paths) if override_path: return PageResolutionResult(True, override_path, None, None) return PageResolutionResult( True, None, self.__get_rel_source_path(possible_path), None) return None
def get_markdown_files(self, dir_): """ Get all the markdown files in a folder, recursively Args: dir_: str, a toplevel folder to walk. """ md_files = OrderedSet() for root, _, files in os.walk(dir_): for name in files: split = os.path.splitext(name) if len(split) == 1: continue if split[1] in ('.markdown', '.md', '.yaml'): md_files.add(os.path.join(root, name)) return md_files
def get_markdown_files(self, dir_): """ Get all the markdown files in a folder, recursively Args: dir_: str, a toplevel folder to walk. """ md_files = OrderedSet() for root, _, files in os.walk(dir_): for name in files: split = os.path.splitext(name) if len(split) == 1: continue if split[1] in ('.markdown', '.md', '.yaml'): md_files.add(os.path.join(root, name)) return md_files
def test_smart_key_default(self): sources = [] sources.append(self._create_src_file('source_a.test', ['symbol_1'])) self.test_ext.sources = sources self.test_ext.setup() self.assertDictEqual(self.test_ext.get_created_symbols(), {sources[0]: OrderedSet(['symbol_1'])})
def parse_config(self, config): """ Override this, making sure to chain up first, if your extension adds its own custom command line arguments, or you want to do any further processing on the automatically added arguments. The default implementation will set attributes on the extension: - 'sources': a set of absolute paths to source files for this extension - 'index': absolute path to the index for this extension Additionally, it will set an attribute for each argument added with `Extension.add_path_argument` or `Extension.add_paths_argument`, with the extension's `Extension.argument_prefix` stripped, and dashes changed to underscores. Args: config: a `config.Config` instance """ prefix = self.argument_prefix self.sources = config.get_sources(prefix) self.smart_sources = [ self._get_smart_filename(s) for s in self.sources] self.index = config.get_index(prefix) self.source_roots = OrderedSet( config.get_paths('%s_source_roots' % prefix)) for arg, dest in list(self.paths_arguments.items()): val = config.get_paths(arg) setattr(self, dest, val) for arg, dest in list(self.path_arguments.items()): val = config.get_path(arg) setattr(self, dest, val) self.formatter.parse_config(config)
def __resolve_patterns(self, source_patterns, from_conf): if source_patterns is None: return OrderedSet() all_files = OrderedSet() for item in source_patterns: item = self.__abspath(item, from_conf) if item in all_files: continue if not os.path.exists(item): all_files |= glob.glob(item) else: all_files.add(item) return all_files
def get_possible_path(self, name): self.__find_package_root() for path in OrderedSet([self.__package_root]) | self.source_roots: possible_path = os.path.join(path, name) if possible_path in self._get_all_sources(): return self._get_smart_filename(possible_path) return None
def __init__(self, source_file, ast, output_path, project_name, meta=None, raw_contents=None): "Banana banana" assert source_file basename = os.path.basename(source_file) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.pre_sorted = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.private_symbols = [] self.typed_symbols = OrderedDict() self.by_parent_symbols = OrderedDict() self.is_stale = True self.formatted_contents = None self.detailed_description = None self.build_path = None self.project_name = project_name self.cached_paths = OrderedSet() meta = meta or {} self.listed_symbols = [] self.symbol_names = [] self.short_description = None self.render_subpages = True self.title = '' self.meta = Schema(Page.meta_schema).validate({}) self.__update_meta(meta) self.__discover_title(meta) self.link = Link(pagename, self.title or name, ref)
def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath(os.path.join( here, 'tmp-markdown-files')) self.private_folder = os.path.abspath(os.path.join( here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join( here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join( here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.private_folder) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) self.dependency_map = {} # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.database = Database(self.private_folder) self.link_resolver = LinkResolver(self.database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.project_name = 'test-project' self.sanitized_name = 'test-project-0.1' self.incremental = False self.tree = Tree(self, self) self.test_ext = TestExtension(self, self) self.core_ext = CoreExtension(self, self) cfg = Config() self.test_ext.parse_toplevel_config(cfg) self.test_ext.parse_config(cfg) self.core_ext.parse_toplevel_config(cfg) self.core_ext.parse_config(cfg) self.subprojects = {} self.is_toplevel = True
def test_smart_key_custom(self): sources = [] sources.append( self._create_src_file('source_a.test', ['custom_key', 'symbol_1'])) self.test_ext.sources = sources self.test_ext.use_custom_key = True self.test_ext.setup() self.assertDictEqual(self.test_ext.get_created_symbols(), {'custom_key': OrderedSet(['symbol_1'])})
def __get_page(self, tree, source_file): page_name = self.__get_rel_source_path(source_file) for path in OrderedSet([self.__package_root]) | self.source_roots: possible_name = os.path.relpath(source_file, path) page = tree.get_pages().get(possible_name) if page: return page, page_name return page, page_name
def __parse_config(self): """ Banana banana """ output = self.config.get_path('output') or None self.sitemap_path = self.config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') if output is not None: self.output = os.path.abspath(output) else: self.output = None self.project_name = self.config.get('project_name', None) self.project_version = self.config.get('project_version', None) self.output_format = self.config.get('output_format') if self.output_format not in ["html"]: error('invalid-config', 'Unsupported output format : %s' % self.output_format) self.__index_file = self.config.get_index() if self.__index_file is None: error('invalid-config', 'index is required') if not os.path.exists(self.__index_file): error('invalid-config', 'The provided index "%s" does not exist' % self.__index_file) cmd_line_includes = self.config.get_paths('include_paths') self.__base_doc_folder = os.path.dirname(self.__index_file) self.include_paths = OrderedSet([self.__base_doc_folder]) self.include_paths |= OrderedSet(cmd_line_includes) self.__create_change_tracker() self.__setup_private_folder() self.__setup_database() self.__create_extensions() if self.__conf_file: self.change_tracker.add_hard_dependency(self.__conf_file)
def __init__(self, app, project): super().__init__(app, project) self.cache = {} self.c_sources = [] self.cache_file = None self.plugin = None self.__elements = {} self.__raw_comment_parser = GtkDocParser(project, section_file_matching=False) self.__plugins = None self.__toplevel_comments = OrderedSet() self.list_plugins_page = None # If we have a plugin with only one element, we render it on the plugin # page. self.unique_feature = None self.__on_index_symbols = [] # Links GTypeName to pagename for other-types so we now where to locate # the symbols on creation. self.__other_types_pages = {}
def __init__(self, app, project): """Constructor for `Extension`. This should never get called directly. Args: project: The `project.Project` instance which documentation is being generated. """ self.project = project self.app = app self.sources = set() self.smart_sources = [] self.index = None self.source_roots = OrderedSet() self._created_symbols = DefaultOrderedDict(OrderedSet) self.__package_root = None self.__toplevel_comments = OrderedSet() self.formatter = self._make_formatter()
def get_dependencies(self): """ Retrieve the set of all dependencies for a given configuration. Returns: utils.utils.OrderedSet: The set of all dependencies for the tracked configuration. """ all_deps = OrderedSet() for key, _ in list(self.__config.items()): if key in self.__cli: continue if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1 - 1]) for key, _ in list(self.__cli.items()): if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1 - 1]) if self.conf_file is not None: all_deps.add(self.conf_file) all_deps.add(self.get_path("sitemap", rel_to_cwd=True)) cwd = os.getcwd() return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def get_pagename(self, name): self.__find_package_root() # Find the longest prefix longest = None for path in OrderedSet([self.__package_root]) | self.source_roots: commonprefix = os.path.commonprefix([path, name]) if commonprefix == path and (longest is None or len(path) > len(longest)): longest = path if longest is not None: return os.path.relpath(name, longest) return name
class Page(object): "Banana banana" meta_schema = {Optional('title'): And(unicode, len), Optional('symbols'): Schema([And(unicode, len)]), Optional('short-description'): And(unicode, len)} resolving_symbol_signal = Signal() formatting_signal = Signal() def __init__(self, source_file, ast, meta=None, raw_contents=None): "Banana banana" assert source_file if os.path.isabs(source_file): basename = os.path.basename(source_file) else: basename = source_file.replace('/', '-') name = os.path.splitext(basename)[0] pagename = '%s.html' % name self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, name) def __getstate__(self): return {'ast': None, 'title': self.title, 'raw_contents': self.raw_contents, 'short_description': self.short_description, 'extension_name': self.extension_name, 'link': self.link, 'meta': self.meta, 'source_file': self.source_file, 'comment': self.comment, 'generated': self.generated, 'is_stale': False, 'formatted_contents': None, 'detailed_description': None, 'output_attrs': None, 'symbols': [], 'typed_symbols': {}, 'subpages': self.subpages, 'symbol_names': self.symbol_names} def resolve_symbols(self, doc_database, link_resolver): """ When this method is called, the page's symbol names are queried from `doc_database`, and added to lists of actual symbols, sorted by symbol class. """ typed_symbols_list = namedtuple( 'TypedSymbolsList', ['name', 'symbols']) self.typed_symbols[Symbol] = typed_symbols_list('FIXME symbols', []) self.typed_symbols[FunctionSymbol] = typed_symbols_list( "Functions", []) self.typed_symbols[CallbackSymbol] = typed_symbols_list( "Callback Functions", []) self.typed_symbols[FunctionMacroSymbol] = typed_symbols_list( "Function Macros", []) self.typed_symbols[ConstantSymbol] = typed_symbols_list( "Constants", []) self.typed_symbols[ExportedVariableSymbol] = typed_symbols_list( "Exported Variables", []) self.typed_symbols[StructSymbol] = typed_symbols_list( "Data Structures", []) self.typed_symbols[EnumSymbol] = typed_symbols_list("Enumerations", []) self.typed_symbols[AliasSymbol] = typed_symbols_list("Aliases", []) self.typed_symbols[SignalSymbol] = typed_symbols_list("Signals", []) self.typed_symbols[PropertySymbol] = typed_symbols_list( "Properties", []) self.typed_symbols[VFunctionSymbol] = typed_symbols_list( "Virtual Methods", []) self.typed_symbols[ClassSymbol] = typed_symbols_list("Classes", []) self.typed_symbols[InterfaceSymbol] = typed_symbols_list("Interfaces", []) all_syms = OrderedSet() for sym_name in self.symbol_names: sym = doc_database.get_symbol(sym_name) self.__query_extra_symbols(sym, all_syms, link_resolver, doc_database) for sym in all_syms: sym.update_children_comments() self.__resolve_symbol(sym, link_resolver) self.symbol_names.add(sym.unique_name) class_syms = self.typed_symbols[ClassSymbol].symbols interface_syms = self.typed_symbols[InterfaceSymbol].symbols struct_syms = self.typed_symbols[StructSymbol].symbols if self.title is None: if class_syms: self.title = class_syms[0].display_name elif interface_syms: self.title = interface_syms[0].display_name elif struct_syms: self.title = struct_syms[0].display_name if self.comment is None: if class_syms and class_syms[0].comment: self.comment = class_syms[0].comment elif interface_syms and interface_syms[0].comment: self.comment = interface_syms[0].comment elif struct_syms and struct_syms[0].comment: self.comment = struct_syms[0].comment # pylint: disable=no-self-use def __fetch_comment(self, sym, doc_database): old_comment = sym.comment new_comment = doc_database.get_comment(sym.unique_name) sym.comment = Comment(sym.unique_name) if new_comment: sym.comment = new_comment elif old_comment: if not old_comment.filename in (ChangeTracker.all_stale_files | ChangeTracker.all_unlisted_files): sym.comment = old_comment def __format_page_comment(self, formatter, link_resolver): if not self.comment: return if self.comment.short_description: self.short_description = formatter.format_comment( self.comment.short_description, link_resolver).strip() if self.short_description.startswith('<p>'): self.short_description = self.short_description[3:-4] if self.comment.title: self.title = formatter.format_comment( self.comment.title, link_resolver).strip() if self.title.startswith('<p>'): self.title = self.title[3:-4] if self.title: self.formatted_contents += '<h1>%s</h1>' % self.title self.formatted_contents += formatter.format_comment( self.comment, link_resolver) def format(self, formatter, link_resolver, output): """ Banana banana """ if not self.title and self.source_file: title = os.path.splitext(self.source_file)[0] self.title = os.path.basename(title).replace('-', ' ') self.formatted_contents = u'' if self.ast: out, diags = cmark.ast_to_html(self.ast, link_resolver) for diag in diags: warn( diag.code, message=diag.message, filename=self.source_file) self.formatted_contents += out else: self.__format_page_comment(formatter, link_resolver) self.output_attrs = defaultdict(lambda: defaultdict(dict)) formatter.prepare_page_attributes(self) Page.formatting_signal(self, formatter) self.__format_symbols(formatter, link_resolver) self.detailed_description =\ formatter.format_page(self)[0] if output: formatter.write_page(self, output) # pylint: disable=no-self-use def get_title(self): """ Banana banana """ return self.title or 'unnamed' def __discover_title(self, meta): if meta is not None and 'title' in meta: self.title = meta['title'] elif self.ast: self.title = cmark.title_from_ast(self.ast) def __format_symbols(self, formatter, link_resolver): for symbol in self.symbols: if symbol is None: continue debug('Formatting symbol %s in page %s' % ( symbol.unique_name, self.source_file), 'formatting') symbol.skip = not formatter.format_symbol(symbol, link_resolver) def __query_extra_symbols(self, sym, all_syms, link_resolver, doc_database): if sym: self.__fetch_comment(sym, doc_database) new_symbols = sum(Page.resolving_symbol_signal(self, sym), []) all_syms.add(sym) for symbol in new_symbols: self.__query_extra_symbols(symbol, all_syms, link_resolver, doc_database) def __resolve_symbol(self, symbol, link_resolver): symbol.resolve_links(link_resolver) symbol.link.ref = "%s#%s" % (self.link.ref, symbol.unique_name) for link in symbol.get_extra_links(): link.ref = "%s#%s" % (self.link.ref, link.id_) tsl = self.typed_symbols[type(symbol)] tsl.symbols.append(symbol) self.symbols.append(symbol) debug('Resolved symbol %s to page %s' % (symbol.display_name, self.link.ref), 'resolution')
class TestDocTree(unittest.TestCase): def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath(os.path.join( here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join( here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join( here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join( here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self) def tearDown(self): self.__remove_tmp_dirs() del self.test_ext del self.core_ext def get_generated_doc_folder(self): return os.path.join(self.__priv_dir, 'generated') def get_base_doc_folder(self): return self.__md_dir def get_private_folder(self): return self.__priv_dir def __parse_sitemap(self, text): path = os.path.join(self.__md_dir, 'sitemap.txt') with io.open(path, 'w', encoding='utf-8') as _: _.write(text) return self.sitemap_parser.parse(path) def __create_md_file(self, name, contents): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: _.write(contents) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) def __create_src_file(self, name, symbols): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: for symbol in symbols: _.write('%s\n' % symbol) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) return path def __remove_src_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __remove_md_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __touch_src_file(self, name): path = os.path.join(self.__md_dir, name) touch(path) def __remove_tmp_dirs(self): shutil.rmtree(self.__md_dir, ignore_errors=True) shutil.rmtree(self.__priv_dir, ignore_errors=True) shutil.rmtree(self.__src_dir, ignore_errors=True) shutil.rmtree(self.__output_dir, ignore_errors=True) shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True) def test_basic(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() # We do not care about ordering self.assertSetEqual( set(pages.keys()), set([u'index.markdown', u'section.markdown'])) index = pages.get('index.markdown') self.assertEqual(index.title, u'My documentation') def test_basic_incremental(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Building from scratch, all pages are stale self.assertSetEqual( set(doc_tree.get_stale_pages()), set([u'index.markdown', u'section.markdown'])) doc_tree.persist() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Nothing changed, no page is stale self.assertSetEqual( set(doc_tree.get_stale_pages()), set({})) # But we still have our pages self.assertSetEqual( set(doc_tree.get_pages()), set([u'index.markdown', u'section.markdown'])) touch(os.path.join(self.__md_dir, u'section.markdown')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) self.assertSetEqual( set(doc_tree.get_stale_pages()), set([u'section.markdown'])) def __assert_extension_names(self, doc_tree, name_map): pages = doc_tree.get_pages() for name, ext_name in name_map.items(): page = pages[name] self.assertEqual(ext_name, page.extension_name) def __assert_stale(self, doc_tree, expected_stale): stale_pages = doc_tree.get_stale_pages() for pagename in expected_stale: self.assertIn(pagename, stale_pages) stale_pages.pop(pagename) self.assertEqual(len(stale_pages), 0) def __create_test_layout(self): inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_x.markdown\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') sources = [] sources.append(self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2'])) sources.append(self.__create_src_file( 'source_b.test', ['symbol_3', 'symbol_4'])) self.test_ext.index = 'test-index.markdown' self.test_ext.sources = sources self.test_ext.setup() sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'core_page.markdown', (u'# My non-extension page\n')) self.__create_md_file( 'test-index.markdown', (u'# My test index\n')) self.__create_md_file( 'test-section.markdown', (u'# My test section\n' '\n' 'Linking to [a generated page](source_a.test)\n')) self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) self.__create_md_file( 'page_y.markdown', (u'# Page Y\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree, sitemap def __update_test_layout(self, doc_tree, sitemap): self.test_ext.reset() self.test_ext.setup() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree def test_extension_basic(self): doc_tree, _ = self.__create_test_layout() self.__assert_extension_names( doc_tree, {u'index.markdown': 'core', u'test-index': 'test-extension', u'test-section.markdown': 'test-extension', u'source_a.test': 'test-extension', u'source_b.test': 'test-extension', u'page_x.markdown': 'test-extension', u'page_y.markdown': 'test-extension', u'core_page.markdown': 'core'}) all_pages = doc_tree.get_pages() self.assertEqual(len(all_pages), 8) self.__assert_stale(doc_tree, all_pages) self.assertNotIn('source_a.test', all_pages['test-index'].subpages) self.assertIn('source_a.test', all_pages['test-section.markdown'].subpages) def test_extension_override(self): self.__create_md_file( 'source_a.test.markdown', (u'# My override\n')) doc_tree, _ = self.__create_test_layout() page = doc_tree.get_pages()['source_a.test'] self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) self.assertEqual( os.path.basename(page.source_file), 'source_a.test.markdown') self.assertEqual( cmark.ast_to_html(page.ast, None), u'<h1>My override</h1>\n') def test_parse_yaml(self): inp = (u'index.markdown\n') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'---\n' 'title: A random title\n' 'symbols: [symbol_1, symbol_2]\n' '...\n' '# My documentation\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() page = pages.get('index.markdown') self.assertEqual( cmark.ast_to_html(page.ast, None), u'<h1>My documentation</h1>\n') self.assertEqual(page.title, u'A random title') self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) def test_empty_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">My documentation</a></p>\n') def test_labeled_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[a label](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">a label</a></p>\n') def test_anchored_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[](index.markdown#subsection)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html#subsection">My documentation</a></p>\n') # pylint: disable=too-many-statements def test_extension_incremental(self): doc_tree, sitemap = self.__create_test_layout() doc_tree.persist() # Here we touch source_a.test, as its symbols were # all contained in a generated page, only that page # should now be stale self.__touch_src_file('source_a.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test'])) doc_tree.persist() # We now touch source_b.test, which symbols are contained # both in a generated page and a user-provided one. # We expect both pages to be stale self.__touch_src_file('source_b.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) doc_tree.persist() # This one is trickier: we unlist symbol_3 from # page_x, which means the symbol should now be # documented in the generated page for source_b.test. # We expect both pages to be stale, and make sure # they contain the right symbols self.__create_md_file( 'page_x.markdown', (u'# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet()) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # Let's make sure the opposite use case works as well, # we relocate symbol_3 in page_x , both page_x and # the generated page for source_b.test should be stale # and the symbols should be back to their original # layout. self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist() # We now move the definition of symbol_3 to source_a.test, # we thus expect the generated page for source_a.test to be # stale because its source changed, same for source_b.test, # and page_x.markdown should be stale as well because the # definition of symbol_3 may have changed. The final # symbol layout should not have changed however. self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2', 'symbol_3']) self.__create_src_file( 'source_b.test', ['symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual( source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # And we rollback again self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2']) self.__create_src_file( 'source_b.test', ['symbol_3', 'symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual( source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # Now we'll try removing page_x altogether self.__remove_md_file('page_x.markdown') inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') new_sitemap = self.__parse_sitemap(inp) doc_tree = self.__update_test_layout(doc_tree, new_sitemap) self.__assert_stale(doc_tree, set(['source_b.test'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # And rollback again self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['page_x.markdown', 'source_b.test'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist()