def parse_config(self, config, toplevel=False): """Parses @config setting up @self state.""" self.sitemap_path = config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') self.include_paths = OrderedSet([]) index_file = config.get_index() if index_file: if not os.path.exists(index_file): error('invalid-config', 'The provided index "%s" does not exist' % index_file) self.include_paths |= OrderedSet([os.path.dirname(index_file)]) self.include_paths |= OrderedSet(config.get_paths('include_paths')) self.is_toplevel = toplevel self.tree = Tree(self, self.app) self.__create_extensions() for extension in list(self.extensions.values()): if toplevel: extension.parse_toplevel_config(config) extension.parse_config(config) if not toplevel and config.conf_file: self.app.change_tracker.add_hard_dependency(config.conf_file) self.extra_asset_folders = OrderedSet(config.get_paths('extra_assets'))
def __update_meta(self, meta): for key, value in meta.items(): try: self.meta.update( Schema(Page.meta_schema).validate( {key.replace('_', '-').lower(): value})) except SchemaError as _: warn( 'invalid-page-metadata', '%s: Invalid metadata: \n%s, discarding metadata' % (self.source_file, str(_))) if not self.meta.get('extra'): self.meta['extra'] = defaultdict() self.title = meta.get('title', self.title) self.thumbnail = meta.get('thumbnail') self.listed_symbols = OrderedSet( meta.get('symbols') or self.symbol_names) self.private_symbols = OrderedSet( meta.get('private-symbols') or self.private_symbols) self.symbol_names = OrderedSet( meta.get('symbols') or self.symbol_names) self.short_description = meta.get('short-description', self.short_description) self.render_subpages = meta.get('render-subpages', self.render_subpages)
def prepare_page_attributes(self, page): """ Banana banana """ page.output_attrs['html']['scripts'] = OrderedSet() page.output_attrs['html']['stylesheets'] = OrderedSet() page.output_attrs['html']['extra_html'] = [] page.output_attrs['html']['extra_footer_html'] = [] if HtmlFormatter.add_anchors: page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'html_assets', 'css.escape.js')) Formatter.prepare_page_attributes(self, page)
def __resolve_patterns(self, source_patterns, from_conf): if source_patterns is None: return OrderedSet() all_files = OrderedSet() for item in source_patterns: item = self.__abspath(item, from_conf) if '*' in item: all_files |= glob.glob(item) else: all_files.add(item) return all_files
def prepare_page_attributes(self, page): """ Banana banana """ self._current_page = page page.output_attrs['html']['scripts'] = OrderedSet() page.output_attrs['html']['stylesheets'] = OrderedSet() page.output_attrs['html']['extra_html'] = [] page.output_attrs['html']['edit_button'] = '' page.output_attrs['html']['extra_footer_html'] = [] if Formatter.add_anchors: page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'assets', 'css.escape.js'))
def __init__(self, source_file, ast, output_path, project_name, meta=None, raw_contents=None): "Banana banana" assert source_file basename = os.path.basename(source_file) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.pre_sorted = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None self.build_path = None self.project_name = project_name self.cached_paths = OrderedSet() meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.render_subpages = meta.get('render-subpages', True) self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, ref)
def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath( os.path.join(here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self)
def get_stale_files(self, all_files, fileset_name): """ Banana banana """ stale = OrderedSet() previous_mtimes = self.mtimes[fileset_name] new_mtimes = defaultdict() for filename in all_files: mtime = get_mtime(filename) prev_mtime = previous_mtimes.pop(filename, None) new_mtimes[filename] = mtime if mtime == prev_mtime: continue stale.add(filename) self.mtimes[fileset_name] = new_mtimes unlisted = set(previous_mtimes.keys()) ChangeTracker.all_stale_files |= stale ChangeTracker.all_unlisted_files |= unlisted return stale, unlisted
def get_pagename(self, name): self.__find_package_root() for path in OrderedSet([self.__package_root]) | self.source_roots: commonprefix = os.path.commonprefix([path, name]) if commonprefix == path: return os.path.relpath(name, path) return name
def __init__(self, app, dependency_map=None, page_map=None): self.app = app self.tree = None self.include_paths = None self.extensions = OrderedDict() self.tag_validators = {} self.project_name = None self.project_version = None self.sanitized_name = None self.sitemap_path = None self.subprojects = {} self.extra_asset_folders = OrderedSet() self.extra_assets = {} if dependency_map is None: self.dependency_map = {} else: self.dependency_map = dependency_map if page_map is None: self.page_map = {} else: self.page_map = page_map if os.name == 'nt': self.datadir = os.path.join(os.path.dirname(__file__), '..', 'share') else: self.datadir = "/usr/share" self.formatted_signal = Signal() self.written_out_signal = Signal() self.is_toplevel = False
def parse_config(self, config): """ Override this, making sure to chain up first, if your extension adds its own custom command line arguments, or you want to do any further processing on the automatically added arguments. The default implementation will set attributes on the extension: - 'sources': a set of absolute paths to source files for this extension - 'index': absolute path to the index for this extension Additionally, it will set an attribute for each argument added with `Extension.add_path_argument` or `Extension.add_paths_argument`, with the extension's `Extension.argument_prefix` stripped, and dashes changed to underscores. Args: config: a `config.Config` instance """ prefix = self.argument_prefix self.sources = config.get_sources(prefix) self.smart_sources = [ self._get_smart_filename(s) for s in self.sources] self.index = config.get_index(prefix) self.source_roots = OrderedSet( config.get_paths('%s_source_roots' % prefix)) for arg, dest in list(self.paths_arguments.items()): val = config.get_paths(arg) setattr(self, dest, val) for arg, dest in list(self.path_arguments.items()): val = config.get_path(arg) setattr(self, dest, val) self.formatter.parse_config(config)
def _resolve_placeholder(self, tree, name, include_paths): self.__find_package_root() if name == '%s-index' % self.argument_prefix: if self.index: path = find_file(self.index, include_paths) if path is None: self.error("invalid-config", "Could not find index file %s" % self.index) return PageResolutionResult(True, path, None, self.extension_name) return PageResolutionResult(True, None, None, self.extension_name) if self.smart_index: for path in OrderedSet([self.__package_root]) | self.source_roots: possible_path = os.path.join(path, name) if possible_path in self._get_all_sources(): override_path = find_file('%s.markdown' % name, include_paths) if override_path: return PageResolutionResult(True, override_path, None, None) return PageResolutionResult( True, None, self.__get_rel_source_path(possible_path), None) return None
def test_smart_key_default(self): sources = [] sources.append(self._create_src_file('source_a.test', ['symbol_1'])) self.test_ext.sources = sources self.test_ext.setup() self.assertDictEqual(self.test_ext.get_created_symbols(), {sources[0]: OrderedSet(['symbol_1'])})
def get_dependencies(self): """ Retrieve the set of all dependencies for a given configuration. Returns: utils.utils.OrderedSet: The set of all dependencies for the tracked configuration. """ all_deps = OrderedSet() for key, _ in list(self.__config.items()): if key in self.__cli: continue if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1 - 1]) for key, _ in list(self.__cli.items()): if key.endswith('sources'): all_deps |= self.get_sources(key[:len('sources') * -1 - 1]) if self.conf_file is not None: all_deps.add(self.conf_file) all_deps.add(self.get_path("sitemap", rel_to_cwd=True)) cwd = os.getcwd() return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def test_parse_yaml(self): inp = (u'index.markdown\n') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'---\n' 'title: A random title\n' 'symbols: [symbol_1, symbol_2]\n' '...\n' '# My documentation\n')) self.tree.parse_sitemap(sitemap) pages = self.tree.get_pages() page = pages.get('index.markdown') out, _ = cmark.ast_to_html(page.ast, None) self.assertEqual( out, u'<h1>My documentation</h1>\n') self.assertEqual(page.title, u'A random title') self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2']))
def __find_structure_pagename(self, node, unique_name, is_class): filename = self.__get_symbol_filename(unique_name, node) if filename != self.__default_page: return filename if self.__symbol_is_relocated(unique_name, None): return self.__default_page if not is_class: sym = self.__class_gtype_structs.get(node.attrib['name']) if sym and sym.filename: return sym.filename filenames = [] for cnode in node: cunique_name = get_symbol_names(cnode)[0] if not cunique_name: continue fname = self.__get_symbol_filename(cunique_name, cnode) if fname != self.__default_page: if cnode.tag == core_ns('constructor'): filenames.insert(0, fname) else: filenames.append(fname) unique_filenames = list(OrderedSet(filenames)) if not filenames: # Did not find any symbols, trying to can get information # about the class structure linked to that object class. nextnode = node.getnext() name = node.attrib['name'] if nextnode is not None and nextnode.tag == core_ns('record'): nextnode_classfor = nextnode.attrib.get( glib_ns('is-gtype-struct-for')) if nextnode_classfor == name: nunique_name = get_symbol_names(nextnode)[0] filename = self.__get_symbol_filename(nunique_name) if filename == self.__default_page: self.warn( "no-location-indication", "No way to determine where %s should land" " putting it to %s." " Document the symbol for smart indexing to work" % (unique_name, os.path.basename(filename))) else: filename = unique_filenames[0] if len(unique_filenames) > 1: self.warn( "no-location-indication", " Going wild here to determine where %s needs to land" " as we could detect the following possibilities: %s." % (unique_name, unique_filenames)) else: self.debug( " No class comment for %s determined that it should" " land into %s with all other class related documentation." % (unique_name, os.path.basename(filename))) return filename
def parse_config(self, config): super(GIExtension, self).parse_config(config) ALL_GIRS.update({os.path.basename(s): s for s in self.sources}) self.c_sources = config.get_sources('gi-c') self.source_roots = OrderedSet(config.get_paths('gi_c_source_roots')) chosen_languages = [l.lower() for l in config.get('languages', [])] languages = [] for lang_type in get_language_classes(): if lang_type.language_name in chosen_languages: languages.append(lang_type()) if languages: self.languages = languages else: self.languages = [ lang_type() for lang_type in get_language_classes() ] # Make sure C always gets formatted first c_language = self.get_language('c') if c_language: self.languages.remove(c_language) self.languages.insert(0, c_language) for gir_file in self.sources: gir_root = etree.parse(gir_file).getroot() cache_nodes(gir_root, ALL_GIRS, self.languages)
def __get_user_symbols(self, tree, index): symbols = self.__get_listed_symbols_in_markdown(tree, index) private_symbols = set() parented_symbols = defaultdict(list) for source_file, symbols_names in list(self._created_symbols.items()): if source_file.endswith(('.markdown', '.md')): continue for symname in symbols_names: symbol = self.app.database.get_symbol(symname) if not symbol.parent_name: continue if symbol.parent_name in symbols: symbols[symbol.unique_name] = symbols[symbol.parent_name] else: parented_symbols[symbol.parent_name].append(symname) page_name = self.__get_page(tree, source_file)[1] comment = self.__get_comment_for_page(source_file, page_name) if not comment: continue for symname in comment.meta.get("private-symbols", OrderedSet()): private_symbols.add(symname) comment_syms, located_parented_symbols = self.__list_symbols_in_comment( comment, parented_symbols, source_file, page_name) if comment_syms: comment.meta['symbols'].extend(located_parented_symbols) symbols.update(comment_syms) return set(symbols.keys()), set(symbols.values()), private_symbols
def __init__(self, source_file, ast, output_path, project_name, meta=None, raw_contents=None): "Banana banana" assert source_file basename = os.path.basename(source_file) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.pre_sorted = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.private_symbols = [] self.typed_symbols = OrderedDict() self.by_parent_symbols = OrderedDict() self.is_stale = True self.formatted_contents = None self.detailed_description = None self.build_path = None self.project_name = project_name self.cached_paths = OrderedSet() meta = meta or {} self.listed_symbols = [] self.symbol_names = [] self.short_description = None self.render_subpages = True self.title = '' self.meta = Schema(Page.meta_schema).validate({}) self.__update_meta(meta) self.__discover_title(meta) self.link = Link(pagename, self.title or name, ref)
def __resolve_patterns(self, source_patterns, from_conf): if source_patterns is None: return OrderedSet() all_files = OrderedSet() for item in source_patterns: item = self.__abspath(item, from_conf) if item in all_files: continue if not os.path.exists(item): all_files |= glob.glob(item) else: all_files.add(item) return all_files
def get_possible_path(self, name): self.__find_package_root() for path in OrderedSet([self.__package_root]) | self.source_roots: possible_path = os.path.join(path, name) if possible_path in self._get_all_sources(): return self._get_smart_filename(possible_path) return None
def test_smart_key_custom(self): sources = [] sources.append( self._create_src_file('source_a.test', ['custom_key', 'symbol_1'])) self.test_ext.sources = sources self.test_ext.use_custom_key = True self.test_ext.setup() self.assertDictEqual(self.test_ext.get_created_symbols(), {'custom_key': OrderedSet(['symbol_1'])})
def __get_page(self, tree, source_file): page_name = self.__get_rel_source_path(source_file) for path in OrderedSet([self.__package_root]) | self.source_roots: possible_name = os.path.relpath(source_file, path) page = tree.get_pages().get(possible_name) if page: return page, page_name return page, page_name
def __parse_config(self): """ Banana banana """ output = self.config.get_path('output') or None self.sitemap_path = self.config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') if output is not None: self.output = os.path.abspath(output) else: self.output = None self.project_name = self.config.get('project_name', None) self.project_version = self.config.get('project_version', None) self.output_format = self.config.get('output_format') if self.output_format not in ["html"]: error('invalid-config', 'Unsupported output format : %s' % self.output_format) self.__index_file = self.config.get_index() if self.__index_file is None: error('invalid-config', 'index is required') if not os.path.exists(self.__index_file): error('invalid-config', 'The provided index "%s" does not exist' % self.__index_file) cmd_line_includes = self.config.get_paths('include_paths') self.__base_doc_folder = os.path.dirname(self.__index_file) self.include_paths = OrderedSet([self.__base_doc_folder]) self.include_paths |= OrderedSet(cmd_line_includes) self.__create_change_tracker() self.__setup_private_folder() self.__setup_database() self.__create_extensions() if self.__conf_file: self.change_tracker.add_hard_dependency(self.__conf_file)
def __init__(self, app, project): """Constructor for `Extension`. This should never get called directly. Args: project: The `project.Project` instance which documentation is being generated. """ self.project = project self.app = app self.sources = set() self.smart_sources = [] self.index = None self.source_roots = OrderedSet() self._created_symbols = DefaultOrderedDict(OrderedSet) self.__package_root = None self.__toplevel_comments = OrderedSet() self.formatter = self._make_formatter()
def __init__(self, source_file, ast, meta=None, raw_contents=None): "Banana banana" assert source_file if os.path.isabs(source_file): basename = os.path.basename(source_file) else: basename = source_file.replace('/', '-') name = os.path.splitext(basename)[0] pagename = '%s.html' % name self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, name)
def get_pagename(self, name): self.__find_package_root() # Find the longest prefix longest = None for path in OrderedSet([self.__package_root]) | self.source_roots: commonprefix = os.path.commonprefix([path, name]) if commonprefix == path and (longest is None or len(path) > len(longest)): longest = path if longest is not None: return os.path.relpath(name, longest) return name
def setup_subpages(pagenames, get_pagename): """Setup subpages for pages with names in @pagenames""" sitemap_pages = sitemap.get_all_sources() for pagename in pagenames: page = self.__all_pages[get_pagename(pagename)] subpages = sitemap_pages.get(get_pagename(pagename), []) page.subpages = OrderedSet(subpages) | page.subpages for subpage_name in page.subpages: if subpage_name not in unlisted_pagenames: subpage = self.__all_pages[subpage_name] if not subpage.meta.get('auto-sort', False): subpage.pre_sorted = True page.subpages -= unlisted_pagenames
def test_extension_override(self): self.__create_md_file('source_a.test.markdown', (u'# My override\n')) _ = self.__create_test_layout() page = self.tree.get_pages()['source_a.test'] self.assertEqual(page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) self.assertEqual(os.path.basename(page.source_file), 'source_a.test.markdown') out, _ = cmark.ast_to_html(page.ast, None) self.assertEqual(out, u'<h1>My override</h1>\n')
def __resolve_patterns(self, source_patterns, from_conf): if source_patterns is None: return OrderedSet() cache_key = self.__get_key(source_patterns, from_conf) all_files = Config.__pattern_cache.get(cache_key, OrderedSet()) if all_files: return all_files for item in source_patterns: item = self.__abspath(item, from_conf) if item in all_files: continue if not os.path.exists(item): all_files |= glob.glob(item) else: all_files.add(item) Config.__pattern_cache[cache_key] = all_files return all_files