def format(self, formatter, link_resolver, output): """ Banana banana """ if not self.title and self.source_file: title = os.path.splitext(self.source_file)[0] self.title = os.path.basename(title).replace('-', ' ') self.formatted_contents = u'' if self.ast: out, diags = cmark.ast_to_html(self.ast, link_resolver) for diag in diags: warn( diag.code, message=diag.message, filename=self.source_file) self.formatted_contents += out else: self.__format_page_comment(formatter, link_resolver) self.output_attrs = defaultdict(lambda: defaultdict(dict)) formatter.prepare_page_attributes(self) Page.formatting_signal(self, formatter) self.__format_symbols(formatter, link_resolver) self.detailed_description =\ formatter.format_page(self)[0] if output: formatter.write_page(self, output)
def __check_since_markers(self, app): with open(self.__symbols_database) as f: prev_symbols = set(json.load(f)) all_symbols = app.database.get_all_symbols() inherited_sinces = set() missing_since_syms = set() for name, sym in all_symbols.items(): if name in prev_symbols: continue if not self.__check_has_since(sym): missing_since_syms.add(sym) else: self.__add_children_with_since(inherited_sinces, sym) for sym in missing_since_syms - inherited_sinces: if sym.comment and sym.comment.filename: filename = sym.comment.filename lineno = sym.comment.endlineno - 1 else: filename = sym.filename lineno = sym.lineno warn('missing-since-marker', message="Missing since marker for %s" % sym.unique_name, filename=filename, lineno=lineno, )
def format(self, formatter, link_resolver, output): """ Banana banana """ if not self.title and self.source_file: title = os.path.splitext(self.source_file)[0] self.title = os.path.basename(title).replace('-', ' ') self.formatted_contents = u'' self.build_path = os.path.join(formatter.get_output_folder(self), self.link.ref) if self.ast: out, diags = cmark.ast_to_html(self.ast, link_resolver) for diag in diags: warn(diag.code, message=diag.message, filename=self.source_file) self.formatted_contents += out if not self.formatted_contents: self.__format_page_comment(formatter, link_resolver) self.output_attrs = defaultdict(lambda: defaultdict(dict)) formatter.prepare_page_attributes(self) self.__format_symbols(formatter, link_resolver) self.detailed_description =\ formatter.format_page(self)[0] if output: formatter.cache_page(self)
def __update_meta(self, meta): for key, value in meta.items(): try: self.meta.update( Schema(Page.meta_schema).validate( {key.replace('_', '-').lower(): value})) except SchemaError as _: warn( 'invalid-page-metadata', '%s: Invalid metadata: \n%s, discarding metadata' % (self.source_file, str(_))) if not self.meta.get('extra'): self.meta['extra'] = defaultdict() self.title = meta.get('title', self.title) self.thumbnail = meta.get('thumbnail') self.listed_symbols = OrderedSet( meta.get('symbols') or self.symbol_names) self.private_symbols = OrderedSet( meta.get('private-symbols') or self.private_symbols) self.symbol_names = OrderedSet( meta.get('symbols') or self.symbol_names) self.short_description = meta.get('short-description', self.short_description) self.render_subpages = meta.get('render-subpages', self.render_subpages)
def __lookup_asset(self, asset, project, page): src = asset.attrib.get('src') if not src: warn('no-image-src', 'Empty image source in %s' % page.source_file) return comps = urllib.parse.urlparse(src) if comps.scheme: return folders = [os.path.dirname(page.source_file)] if not page.generated \ else [] folders += [ os.path.join(folder, os.path.pardir) for folder in project.extra_asset_folders ] for folder in folders: path = os.path.abspath(os.path.join(folder, src)) if os.path.exists(path): output_folder = os.path.join(self.get_output_folder(page), os.path.dirname(page.link.ref)) project.extra_assets[os.path.join(output_folder, src)] = path asset.attrib['src'] = os.path.join(output_folder, src) return warn('bad-image-src', ('In %s, a local assets refers to an unknown source (%s). ' 'It should be available in one of these locations: %s') % (page.source_file, src, str(folders)))
def page_from_raw_text(self, source_file, contents, extension_name): """ Banana banana """ raw_contents = contents meta = {} if contents.startswith('---\n'): split = contents.split('\n...\n', 1) if len(split) == 2: contents = split[1] try: blocks = yaml.load_all(split[0], Loader=yaml.FullLoader) for block in blocks: if block: meta.update(block) except ConstructorError as exception: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (source_file, str(exception))) output_path = os.path.dirname(os.path.relpath( source_file, next(iter(self.project.include_paths)))) ast = cmark.hotdoc_to_ast(contents, self, source_file) return Page(source_file, False, self.project.sanitized_name, extension_name, source_file=source_file, ast=ast, meta=meta, raw_contents=raw_contents, output_path=output_path)
def page_from_raw_text(self, source_file, contents): """ Banana banana """ raw_contents = contents meta = {} if contents.startswith('---\n'): split = contents.split('\n...\n', 1) if len(split) == 2: contents = split[1] try: blocks = yaml.load_all(split[0]) for block in blocks: if block: meta.update(block) except ConstructorError as exception: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (source_file, str(exception))) output_path = os.path.dirname(os.path.relpath( source_file, iter(self.__include_paths).next())) ast = cmark.hotdoc_to_ast(contents, self) return Page(source_file, ast, output_path, meta=meta, raw_contents=raw_contents)
def warn(self, code, message): """ Shortcut function for `loggable.warn` Args: code: see `utils.loggable.warn` message: see `utils.loggable.warn` """ warn(code, message)
def write_page(self, page, output): root = etree.HTML(unicode(page.detailed_description)) id_nodes = {n.attrib['id']: "".join([x for x in n.itertext()]) for n in root.xpath('.//*[@id]')} section_numbers = self.__init_section_numbers(root) targets = root.xpath( './/*[self::h1 or self::h2 or self::h3 or ' 'self::h4 or self::h5 or self::img]') for target in targets: section_number = self.__update_section_number( target, section_numbers) if 'id' in target.attrib: continue if target.tag == 'img': text = target.attrib.get('alt') else: text = "".join([x for x in target.itertext()]) if not text: continue id_ = id_from_text(text) ref_id = id_ index = 1 while id_ in id_nodes: id_ = '%s%s' % (ref_id, index) index += 1 if section_number: target.text = '%s %s' % (section_number, target.text or '') target.attrib['id'] = id_ id_nodes[id_] = text empty_links = root.xpath('.//a[not(text()) and not(*)]') for link in empty_links: href = link.attrib.get('href') if href and href.startswith('#'): title = id_nodes.get(href.strip('#')) if title: link.text = title else: warn('bad-local-link', "Empty anchor link to %s in %s points nowhere" % (href, page.source_file)) link.text = "FIXME broken link to %s" % href page.detailed_description = lxml.html.tostring( root, doctype="<!DOCTYPE html>", encoding='unicode', include_meta_content_type=True) return Formatter.write_page(self, page, output)
def get_clang_headers(): version = subprocess.check_output(['llvm-config', '--version']).strip() prefix = subprocess.check_output(['llvm-config', '--prefix']).strip() for lib in ['lib', 'lib64']: p = os.path.join(prefix, lib, 'clang', version, 'include') if os.path.exists(p): return p warn('clang-headers-not-found', CLANG_HEADERS_WARNING)
def __parse_yaml_comment(self, comment, filename): res = {} try: blocks = yaml.load_all(comment.raw_comment, Loader=yaml.SafeLoader) for block in blocks: if block: res.update(block) except (ConstructorError, yaml.parser.ParserError) as exception: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (filename, str(exception))) return res
def __dummy_system_message(instance, level, message, *children, **kwargs): if comment.lineno != -1 and 'line' in kwargs: lineno = comment.lineno + kwargs.get('line') else: lineno = -1 warn( 'python-doc-issue', message=message, filename=comment.filename, lineno=lineno)
def get_clang_headers(): version = subprocess.check_output(['llvm-config', '--version']).strip().decode() prefix = subprocess.check_output(['llvm-config', '--prefix']).strip().decode() for lib in ['lib', 'lib64']: p = os.path.join(prefix, lib, 'clang', version, 'include') if os.path.exists(p): return p warn('clang-headers-not-found', CLANG_HEADERS_WARNING)
def __format_content(self, formatter, link_resolver): if self.ast: out, diags = cmark.ast_to_html(self.ast, link_resolver) for diag in diags: warn( diag.code, message=diag.message, filename=self.source_file or self.name) self.formatted_contents += out if not self.formatted_contents: self.__format_page_comment(formatter, link_resolver)
def __init__(self, source_file, ast, output_path, project_name, meta=None, raw_contents=None): "Banana banana" assert source_file basename = os.path.basename(source_file) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.pre_sorted = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None self.build_path = None self.project_name = project_name self.cached_paths = OrderedSet() meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.render_subpages = meta.get('render-subpages', True) self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, ref)
def _formatting_page_cb(self, formatter: Formatter, page: Page) -> None: ''' Replace Meson refman tags Links of the form [[function]] are automatically replaced with valid links to the correct URL. To reference objects / types use the [[@object]] syntax. ''' link_regex = re.compile( r'(\[\[#?@?([ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\.)*[ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\]\])(.)?', re.MULTILINE) for m in link_regex.finditer(page.formatted_contents): i = m.group(1) obj_id: str = i[2:-2] obj_id = re.sub(r'[ \n\t]', '', obj_id) # Remove whitespaces # Marked as inside a code block? in_code_block = False if obj_id.startswith('#'): in_code_block = True obj_id = obj_id[1:] if obj_id not in self._data: warn( 'unknown-refman-link', f'{Path(page.name).name}: Unknown Meson refman link: "{obj_id}"' ) continue # Just replaces [[!file.id]] paths with the page file (no fancy HTML) if obj_id.startswith('!'): page.formatted_contents = page.formatted_contents.replace( i, self._data[obj_id]) continue # Fancy links for functions and methods text = obj_id if text.startswith('@'): text = text[1:] elif in_code_block: if m.group(3) != '(': text = text + '()' else: text = text + '()' if not in_code_block: text = f'<code>{text}</code>' link = f'<a href="{self._data[obj_id]}"><ins>{text}</ins></a>' page.formatted_contents = page.formatted_contents.replace( i, link, 1)
def __formatting_page_cb(self, formatter, page): # Only allow editing markdown pages if page.generated: return root = self.__get_repo_root(page) if not root: warn('source-not-in-git-repo', 'File %s not in a git repository' % (page.source_file)) edit_link = self.__repo + '/edit/' + self.__branch + \ '/' + os.path.relpath(page.source_file, root) page.output_attrs['html']['edit_button'] = \ '<a href=%s data-hotdoc-role="edit-button">' \ 'Edit on github</a>' % edit_link
def __update_links(self, page, doc_root, id_nodes): rel_path = os.path.join(self.get_output_folder(page), page.link.ref) links = doc_root.xpath('.//*[@data-hotdoc-role="main"]//a') for link in links: href = link.attrib.get('href') if href and href.startswith('#'): if not link.text and not link.getchildren(): id_node = id_nodes.get(href.strip('#')) if id_node is not None: link.text = ''.join([x for x in id_node.itertext()]) else: warn('bad-local-link', "Empty anchor link to %s in %s points nowhere" % (href, page.name)) link.text = "FIXME broken link to %s" % href link.attrib["href"] = rel_path + href
def create_symbol(self, type_, **kwargs): """ Banana banana """ unique_name = kwargs.get('unique_name') if not unique_name: unique_name = kwargs.get('display_name') kwargs['unique_name'] = unique_name filename = kwargs.get('filename') if filename: filename = os.path.abspath(filename) kwargs['filename'] = os.path.abspath(filename) if unique_name in self.__symbols and not type_ == ProxySymbol: warn('symbol-redefined', "%s(unique_name=%s, filename=%s, project=%s)" " has already been defined: %s" % (type_.__name__, unique_name, filename, kwargs.get('project_name'), self.get_symbol(unique_name))) return None aliases = kwargs.pop('aliases', []) alias_symbols = [] for alias in aliases: alias_symbols.append( self.create_symbol(ProxySymbol, unique_name=alias, target=unique_name)) symbol = type_() debug('Created symbol with unique name %s' % unique_name, 'symbols') for key, value in list(kwargs.items()): setattr(symbol, key, value) symbol.aliases += alias_symbols if not isinstance(symbol, ProxySymbol): self.__symbols[unique_name] = symbol self.__aliased[unique_name].extend(aliases) for alias in self.__aliased[unique_name]: self.__aliases[alias] = symbol return symbol
def __redirect_if_needed(self, formatter, link_resolver): redirect = self.meta.get("redirect") if not redirect: return False link = link_resolver.get_named_link(redirect) if link: if formatter.extension.project.is_toplevel: page_path = self.link.ref else: page_path = self.project_name + '/' + self.link.ref self_dir = os.path.dirname(page_path) if self_dir == self.link.ref: self_dir = "" self.meta["redirect"] = os.path.relpath(link.ref, self_dir) else: warn('markdown-bad-link', "Bad redirect link '%s' in page: %s" % (redirect, self.name)) return True
def __include_file_cb(self, include_path, line_ranges, symbol_name): if not include_path.endswith(".c") or not symbol_name: return None if not line_ranges: line_ranges = [(1, -1)] symbol = self.app.database.get_symbol(symbol_name) if symbol and symbol.filename != include_path: symbol = None if not symbol: scanner = ClangScanner(self.app, self.project, self) scanner.scan([include_path], self.flags, self.app.incremental, True, ['*.c', '*.h']) symbol = self.app.database.get_symbol(symbol_name) if not symbol: warn( 'bad-c-inclusion', "Trying to include symbol %s but could not be found in " "%s" % (symbol_name, include_path)) return None res = '' for n, (start, end) in enumerate(line_ranges): if n != 0: res += "\n...\n" start += symbol.extent_start - 2 if end > 0: end += (symbol.extent_start - 1) # We are inclusive here else: end = symbol.extent_end with open(include_path, "r") as _: res += "\n".join(_.read().split("\n")[start:end]) if res: return res, 'c' return None
def __download_theme(self, uri): sha = urllib.parse.parse_qs(uri.query).get('sha256') cachedir = appdirs.user_cache_dir("hotdoc", "hotdoc") os.makedirs(cachedir, exist_ok=True) if sha: sha = sha[0] if sha and os.path.isdir(os.path.join(cachedir, "themes", sha)): return os.path.join(cachedir, "themes", sha) print("Downloading %s" % uri.geturl()) tarball = os.path.join(cachedir, "themes", os.path.basename(uri.path)) os.makedirs(os.path.dirname(tarball), exist_ok=True) try: urlretrieve(uri.geturl(), tarball, _download_progress_cb) except (TimeoutError, urllib.error.HTTPError) as exce: warn( 'download-theme-error', "Error downloading %s: %s - Using default them" % (tarball, exce)) return 'default' if not sha: with open(tarball) as file_: sha = hashlib.sha256().update(file_.read()).hexdigest() themepath = os.path.join(cachedir, "themes", sha) try: os.makedirs(os.path.join(themepath)) with tarfile.open(tarball) as file_: file_.extractall(themepath) except tarfile.ReadError: warn( 'download-theme-error', "%s is not a supported tarball - Using default theme" % themepath) os.rmdir(themepath) return 'default' return themepath
def __formatting_page_cb(self, formatter, page): prism_theme = Formatter.theme_meta.get('prism-theme', 'prism') prism_theme_path = '%s.css' % os.path.join(HERE, 'prism', 'themes', prism_theme) if os.path.exists(prism_theme_path): page.output_attrs['html']['stylesheets'].add(prism_theme_path) else: warn('syntax-invalid-theme', 'Prism has no theme named %s' % prism_theme) page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'prism', 'components', 'prism-core.js')) page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'prism', 'plugins', 'autoloader', 'prism-autoloader.js')) page.output_attrs['html']['scripts'].add( os.path.join(HERE, 'prism_autoloader_path_override.js')) folder = os.path.join('html', 'assets', 'prism_components') self.__asset_folders.add(folder)
def __init__(self, source_file, ast, meta=None, raw_contents=None): "Banana banana" assert source_file if os.path.isabs(source_file): basename = os.path.basename(source_file) else: basename = source_file.replace('/', '-') name = os.path.splitext(basename)[0] pagename = '%s.html' % name self.ast = ast self.extension_name = None self.source_file = source_file self.raw_contents = raw_contents self.comment = None self.generated = False self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.typed_symbols = {} self.is_stale = True self.formatted_contents = None self.detailed_description = None meta = meta or {} try: self.meta = Schema(Page.meta_schema).validate(meta) except SchemaError as _: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s' % (self.source_file, str(_))) self.meta = meta self.symbol_names = OrderedSet(meta.get('symbols') or []) self.short_description = meta.get('short-description') self.title = None self.__discover_title(meta) self.link = Link(pagename, self.title or name, name)
def __include_file_cb(self, include_path, line_ranges, symbol_name): if not include_path.endswith(".c") or not symbol_name: return None if not line_ranges: line_ranges = [(1, -1)] symbol = self.doc_repo.doc_database.get_symbol(symbol_name) if symbol and symbol.filename != include_path: symbol = None if not symbol: scanner = ClangScanner(self.doc_repo, self) scanner.scan([include_path], CExtension.flags, self.doc_repo.incremental, True, ['*.c', '*.h']) symbol = self.doc_repo.doc_database.get_symbol(symbol_name) if not symbol: warn('bad-c-inclusion', "Trying to include symbol %s but could not be found in " "%s" % (symbol_name, include_path)) return None res = '' for n, (start, end) in enumerate(line_ranges): if n != 0: res += "\n...\n" start += symbol.extent_start - 2 if end > 0: end += (symbol.extent_start - 1) # We are inclusive here else: end = symbol.extent_end with open(include_path, "r") as _: res += "\n".join(_.read().split("\n")[start:end]) if res: return res, 'c' return None
def format(self, formatter, link_resolver, root, output): """ Banana banana """ if not self.title and self.source_file: title = os.path.splitext(self.source_file)[0] self.title = os.path.basename(title).replace('-', ' ') self.formatted_contents = u'' self.build_path = os.path.join( os.path.relpath(formatter.get_output_folder(), 'html').lstrip('./'), self.link.ref) Link.relativize_link_signal.connect(self.__relativize_link_cb) if self.ast: out, diags = cmark.ast_to_html(self.ast, link_resolver) for diag in diags: warn( diag.code, message=diag.message, filename=self.source_file) self.formatted_contents += out else: self.__format_page_comment(formatter, link_resolver) self.output_attrs = defaultdict(lambda: defaultdict(dict)) formatter.prepare_page_attributes(self) Page.formatting_signal(self, formatter) self.__format_symbols(formatter, link_resolver) self.detailed_description =\ formatter.format_page(self)[0] Link.relativize_link_signal.disconnect(self.__relativize_link_cb) if output: formatter.write_page(self, root, output)
def __parse_tag(self, name, desc): if name.lower() == "since": return self.__parse_since_tag(name, desc) if name.lower() == "returns": return self.__parse_returns_tag(name, desc) if name.lower() == "return value": return self.__parse_returns_tag("returns", desc) if name.lower() == "stability": return self.__parse_stability_tag("stability", desc) if name.lower() == "deprecated": return self.__parse_deprecated_tag("deprecated", desc) if name.lower() == "topic": return self.__parse_topic_tag("topic", desc) validator = self.project.tag_validators.get(name) if not validator: warn('gtk-doc', "FIXME no tag validator") return None if not validator.validate(desc): warn('gtk-doc', "invalid value for tag %s : %s" % (name, desc)) return None return Tag(name=name, description=desc, value=desc)
def get_link(self, link_resolver): """ Banana banana """ res = link_resolver.resolving_link_signal(self) if not res: ref = self.ref else: ref = res[0] or self.ref if self.__mandatory and not self.__warned: if not ref: warn('mandatory-link-not-found', 'Mandatory link %s could not be resolved' % self) self.__warned = True if not res: return ref, None if res[1]: return ref, dict_to_html_attrs(res[1]) return ref, None
def page_from_raw_text(self, source_file, contents): """ Banana banana """ raw_contents = contents meta = {} if contents.startswith('---\n'): split = contents.split('\n...\n', 1) if len(split) == 2: contents = split[1] try: blocks = yaml.load_all(split[0]) for block in blocks: if block: meta.update(block) except ConstructorError as exception: warn( 'invalid-page-metadata', '%s: Invalid metadata: \n%s' % (source_file, str(exception))) ast = cmark.hotdoc_to_ast(contents, self) return Page(source_file, ast, meta=meta, raw_contents=raw_contents)
def __clone_and_update_repo(self): cachedir = appdirs.user_cache_dir("hotdoc", "hotdoc") repo_url, repo_path = _split_repo_url(self.__repository) if repo_url is None or repo_path is None: warn( "git-error", "%s doesn't seem to contain a repository URL" % (self.__repository)) return None sanitize = str.maketrans('/@:', '___') repo = os.path.join(cachedir, 'git-upload', repo_url.translate(sanitize)) try: call(['git', 'rev-parse', '--show-toplevel'], cwd=repo, stderr=subprocess.STDOUT) except (subprocess.CalledProcessError, FileNotFoundError): print("cloning %s" % repo_url) try: subprocess.check_call(['git', 'clone', repo_url, repo]) except subprocess.CalledProcessError as exc: warn( "git-error", "Could not clone %s in %s: %s" % (repo_url, repo, exc.output)) return None try: call(['git', 'checkout', self.__remote_branch[1]], cwd=repo, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: warn( "git-error", "Could not checkout branch %s in %s: %s" % (self.__remote_branch[1], repo, exc.output)) return None try: call(['git', 'pull'], cwd=repo, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: warn( "git-error", "Could not update %s (in %s) from remote %s" % (repo_url, repo, exc.output)) return None return os.path.join(repo, repo_path)
def cache_nodes(gir_root, all_girs): ''' Identify and store all the gir symbols the symbols we will document may link to, or be typed with ''' ns_node = gir_root.find('./{%s}namespace' % NS_MAP['core']) id_prefixes = ns_node.attrib['{%s}identifier-prefixes' % NS_MAP['c']] sym_prefixes = ns_node.attrib['{%s}symbol-prefixes' % NS_MAP['c']] id_key = '{%s}identifier' % NS_MAP['c'] for node in gir_root.xpath('.//*[@c:identifier]', namespaces=NS_MAP): make_translations(node.attrib[id_key], node) id_type = c_ns('type') glib_type = glib_ns('type-name') class_tag = core_ns('class') callback_tag = core_ns('callback') interface_tag = core_ns('interface') for node in gir_root.xpath( './/*[not(self::core:type) and not (self::core:array)][@c:type or @glib:type-name]', namespaces=NS_MAP): try: name = node.attrib[id_type] except KeyError: name = node.attrib[glib_type] make_translations(name, node) gi_name = '.'.join(get_gi_name_components(node)) ALL_GI_TYPES[gi_name] = get_klass_name(node) if node.tag in (class_tag, interface_tag): __update_hierarchies(ns_node.attrib.get('name'), node, gi_name) make_translations('%s::%s' % (name, name), node) __generate_smart_filters(id_prefixes, sym_prefixes, node) elif node.tag in (callback_tag, ): ALL_CALLBACK_TYPES.add(node.attrib[c_ns('type')]) for field in gir_root.xpath('.//self::core:field', namespaces=NS_MAP): unique_name = get_field_c_name(field) make_translations(unique_name, field) for node in gir_root.xpath('.//core:property', namespaces=NS_MAP): name = '%s:%s' % (get_klass_name( node.getparent()), node.attrib['name']) make_translations(name, node) for node in gir_root.xpath('.//glib:signal', namespaces=NS_MAP): name = '%s::%s' % (get_klass_name( node.getparent()), node.attrib['name']) make_translations(name, node) for node in gir_root.xpath('.//core:virtual-method', namespaces=NS_MAP): name = get_symbol_names(node)[0] make_translations(name, node) for inc in gir_root.findall('./core:include', namespaces=NS_MAP): inc_name = inc.attrib["name"] inc_version = inc.attrib["version"] gir_file = __find_gir_file('%s-%s.gir' % (inc_name, inc_version), all_girs) if not gir_file: warn('missing-gir-include', "Couldn't find a gir for %s-%s.gir" % (inc_name, inc_version)) continue if gir_file in __PARSED_GIRS: continue __PARSED_GIRS.add(gir_file) inc_gir_root = etree.parse(gir_file).getroot() cache_nodes(inc_gir_root, all_girs)
def write_page(self, page, build_root, output): root = etree.HTML(unicode(page.detailed_description)) id_nodes = {n.attrib['id']: "".join([x for x in n.itertext()]) for n in root.xpath('.//*[@id]')} section_numbers = self.__init_section_numbers(root) targets = root.xpath( './/*[self::h1 or self::h2 or self::h3 or ' 'self::h4 or self::h5 or self::img]') for target in targets: section_number = self.__update_section_number( target, section_numbers) if 'id' in target.attrib: continue if target.tag == 'img': text = target.attrib.get('alt') else: text = "".join([x for x in target.itertext()]) if not text: continue id_ = id_from_text(text) ref_id = id_ index = 1 while id_ in id_nodes: id_ = '%s%s' % (ref_id, index) index += 1 if section_number: target.text = '%s %s' % (section_number, target.text or '') target.attrib['id'] = id_ id_nodes[id_] = text empty_links = root.xpath('.//a[not(text()) and not(*)]') for link in empty_links: href = link.attrib.get('href') if href and href.startswith('#'): title = id_nodes.get(href.strip('#')) if title: link.text = title else: warn('bad-local-link', "Empty anchor link to %s in %s points nowhere" % (href, page.source_file)) link.text = "FIXME broken link to %s" % href page.detailed_description = lxml.html.tostring( root, doctype="<!DOCTYPE html>", encoding='unicode', include_meta_content_type=True) full_path = Formatter.write_page(self, page, build_root, output) images = root.xpath('.//img') # All required assets should now be in place for img in images: src = img.attrib.get('src') if not src: warn('no-image-src', 'Empty image source in %s' % page.source_file) continue comps = urlparse.urlparse(src) if comps.scheme: continue path = os.path.abspath(os.path.join( os.path.dirname(full_path), src)) if not os.path.exists(path): warn('bad-image-src', ('In %s, a local image refers to an unknown source (%s). ' 'It should be available in the build folder, at %s') % (page.source_file, src, path)) continue return full_path
def scan(self, filenames, options, full_scan, full_scan_patterns, fail_fast=False, all_sources=None): if all_sources is None: self.__all_sources = [] else: self.__all_sources = all_sources index = cindex.Index.create() flags = cindex.TranslationUnit.PARSE_INCOMPLETE | cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD info('scanning %d C source files' % len(filenames)) self.filenames = filenames # FIXME: er maybe don't do that ? args = ["-Wno-attributes"] args.append("-isystem%s" % CLANG_HEADERS) args.extend(options) self.symbols = {} self.parsed = set({}) debug('CFLAGS %s' % ' '.join(args)) header_guarded = set() for filename in self.filenames: if filename in self.parsed: continue do_full_scan = any( fnmatch(filename, p) for p in full_scan_patterns) if do_full_scan: debug('scanning %s' % filename) tu = index.parse(filename, args=args, options=flags) for diag in tu.diagnostics: s = diag.format() warn('clang-diagnostic', 'Clang issue : %s' % str(diag)) self.__parse_file(filename, tu, full_scan) if (cindex.conf.lib.clang_isFileMultipleIncludeGuarded( tu, tu.get_file(filename))): header_guarded.add(filename) for include in tu.get_includes(): fname = os.path.abspath(str(include.include)) if (cindex.conf.lib.clang_isFileMultipleIncludeGuarded( tu, tu.get_file(fname))): if fname in self.filenames: header_guarded.add(fname) self.__parse_file(fname, tu, full_scan) if not full_scan: comment_parser = GtkDocParser(self.project) CCommentExtractor(self.__doc_db, comment_parser).parse_comments(filenames) return True
def parse_comment(self, comment, filename, lineno, endlineno, include_paths=None, stripped=False): """ Returns a Comment given a string """ if not stripped and not self.__validate_c_comment(comment.strip()): return None comment = unicode(comment.decode('utf8')) title_offset = 0 column_offset = 0 raw_comment = comment if not stripped: try: while comment[column_offset * -1 - 1] != '\n': column_offset += 1 except IndexError: column_offset = 0 comment, title_offset = self.__strip_comment(comment) split = re.split(r'\n[\W]*\n', comment, maxsplit=1) try: block_name, parameters, annotations = \ self.__parse_title_and_parameters(split[0]) except HotdocSourceException as _: warn('gtk-doc-bad-syntax', message=_.message, filename=filename, lineno=lineno + title_offset) return None params_offset = 0 for param in parameters: param.filename = filename param.lineno = lineno param_offset = param.line_offset param.line_offset = title_offset + params_offset + 1 params_offset += param_offset param.col_offset = column_offset if not block_name: return None description_offset = 0 description = "" tags = [] if len(split) > 1: n_lines = len(comment.split('\n')) description_offset = (title_offset + n_lines - len(split[1].split('\n'))) description, tags = self.__parse_description_and_tags(split[1]) title = None short_description = None actual_parameters = {} for param in parameters: if param.name.lower() == 'short_description': short_description = param elif param.name.lower() == 'title': title = param else: actual_parameters[param.name] = param annotations = {annotation.name: annotation for annotation in annotations} tags = {tag.name.lower(): tag for tag in tags} block = Comment(name=block_name, filename=filename, lineno=lineno, endlineno=endlineno, annotations=annotations, params=actual_parameters, description=description, short_description=short_description, title=title, tags=tags, raw_comment=raw_comment) block.line_offset = description_offset block.col_offset = column_offset return block
def __formatted_cb(self, app): repo = self.__local_repo if not repo: if not self.__repository or not self.__activate: return repo = self.__clone_and_update_repo() if not repo: return try: call(['git', 'rev-parse', '--show-toplevel'], cwd=repo, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exception: warn( 'no-local-repository', "Specified local repository %s does not exist, " "not uploading. (%s)" % (repo, str(exception))) if subprocess.call(['git', 'diff-index', '--quiet', 'HEAD'], cwd=repo) != 0: warn( 'dirty-local-repository', "Specified local repository %s is dirty, not uploading." % (repo)) return html_path = os.path.join(app.output, "html/") print("Uploading %s to %s!" % (html_path, repo)) print("Removing previous assets %s" % os.path.join(repo, "assets/")) shutil.rmtree(os.path.join(repo, "assets/"), ignore_errors=True) print("Removing previous html files") # pylint: disable=unused-variable for root, subdirs, files in os.walk(repo): for filename in files: if filename.endswith('.html'): os.remove(os.path.join(root, filename)) print("Copying newly built files") for filename in os.listdir(html_path): built_f = os.path.join(html_path, filename) copy_dest = os.path.join(repo, filename) recursive_overwrite(built_f, copy_dest) call(['git', 'add', filename], cwd=repo) if self.__copy_only: return if subprocess.call(['git', 'diff-index', '--quiet', 'HEAD'], cwd=repo) == 0: print("No changes to push") return print("Committing %s" % self.__commit_message) subprocess.check_call( ['git', 'commit', '-a', '-m', self.__commit_message], cwd=repo) print("Pushing to %s" % '/'.join(self.__remote_branch)) subprocess.check_call(['git', 'push'] + self.__remote_branch, cwd=repo)
def scan(self, filenames, options, incremental, full_scan, full_scan_patterns, fail_fast=False): index = clang.cindex.Index.create() flags = clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD info('scanning %d C source files' % len(filenames)) self.filenames = filenames # FIXME: er maybe don't do that ? args = ["-Wno-attributes"] args.append ("-isystem%s" % get_clang_headers()) args.extend (options) self.symbols = {} self.parsed = set({}) debug('CFLAGS %s' % ' '.join(args)) header_guarded = set() for filename in self.filenames: if filename in self.parsed: continue do_full_scan = any(fnmatch(filename, p) for p in full_scan_patterns) if do_full_scan: debug('scanning %s' % filename) tu = index.parse(filename, args=args, options=flags) for diag in tu.diagnostics: warn('clang-diagnostic', 'Clang issue : %s' % str(diag)) self.__parse_file (filename, tu, full_scan) if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(filename))): header_guarded.add(filename) for include in tu.get_includes(): fname = os.path.abspath(str(include.include)) if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(fname))): if fname in self.filenames: header_guarded.add(fname) self.__parse_file (fname, tu, full_scan) if not full_scan: for filename in filenames: with open (filename, 'r') as f: skip_next_symbol = filename in header_guarded debug('Getting comments in %s' % filename) cs = get_comments (filename) for c in cs: if c[4]: block = self.__raw_comment_parser.parse_comment(c[0], c[1], c[2], c[3], self.doc_repo.include_paths) if block is not None: self.doc_repo.doc_database.add_comment(block) elif not skip_next_symbol: if filename.endswith('.h'): self.__create_macro_from_raw_text(c) else: skip_next_symbol = False return True
def comment_to_ast(self, comment, link_resolver): """ Given a gtk-doc comment string, returns an opaque PyCapsule containing the document root. This is an optimization allowing to parse the docstring only once, and to render it multiple times with `ast_to_html`, links discovery and most of the link resolution being lazily done in that second phase. If you don't care about performance, you should simply use `translate`. Args: text: unicode, the docstring to parse. link_resolver: hotdoc.core.links.LinkResolver, an object which will be called to retrieve `hotdoc.core.links.Link` objects. Returns: capsule: A PyCapsule wrapping an opaque C pointer, which can be passed to `ast_to_html` afterwards. diagnostics: A list of diagnostics as output by the gtk-doc cmark extension """ assert comment is not None text = comment.description if (GtkDocStringFormatter.remove_xml_tags or comment.filename in GtkDocStringFormatter.gdbus_codegen_sources): text = re.sub('<.*?>', '', text) if GtkDocStringFormatter.escape_html: text = cgi.escape(text) ast, diagnostics = cmark.gtkdoc_to_ast(text, link_resolver) for diag in diagnostics: if (comment.filename and comment.filename not in GtkDocStringFormatter.gdbus_codegen_sources): column = diag.column + comment.col_offset if diag.lineno == 0: column += comment.initial_col_offset lines = text.split('\n') line = lines[diag.lineno] i = 0 while line[i] == ' ': i += 1 column += i - 1 if diag.lineno > 0 and any([c != ' ' for c in lines[diag.lineno - 1]]): column += 1 lineno = -1 if comment.lineno != -1: lineno = (comment.lineno - 1 + comment.line_offset + diag.lineno) warn( diag.code, message=diag.message, filename=comment.filename, lineno=lineno, column=column) return ast
def __cache_nodes(self, gir_root): ns_node = gir_root.find('./{%s}namespace' % self.__nsmap['core']) id_prefixes = ns_node.attrib['{%s}identifier-prefixes' % self.__nsmap['c']] sym_prefixes = ns_node.attrib['{%s}symbol-prefixes' % self.__nsmap['c']] id_key = '{%s}identifier' % self.__nsmap['c'] for node in gir_root.xpath( './/*[@c:identifier]', namespaces=self.__nsmap): self.__node_cache[node.attrib[id_key]] = node id_type = '{%s}type' % self.__nsmap['c'] class_tag = '{%s}class' % self.__nsmap['core'] interface_tag = '{%s}interface' % self.__nsmap['core'] for node in gir_root.xpath( './/*[not(self::core:type) and not (self::core:array)][@c:type]', namespaces=self.__nsmap): name = node.attrib[id_type] self.__node_cache[name] = node if node.tag in [class_tag, interface_tag]: gi_name = '.'.join(self.__get_gi_name_components(node)) self.__class_nodes[gi_name] = node get_type_function = node.attrib.get('{%s}get-type' % self.__nsmap['glib']) self.__get_type_functions.add(get_type_function) self.__node_cache['%s::%s' % (name, name)] = node self.__generate_smart_filters(id_prefixes, sym_prefixes, node) for node in gir_root.xpath( './/core:property', namespaces=self.__nsmap): name = '%s:%s' % (self.__get_klass_name(node.getparent()), node.attrib['name']) self.__node_cache[name] = node for node in gir_root.xpath( './/glib:signal', namespaces=self.__nsmap): name = '%s::%s' % (self.__get_klass_name(node.getparent()), node.attrib['name']) self.__node_cache[name] = node for node in gir_root.xpath( './/core:virtual-method', namespaces=self.__nsmap): name = '%s:::%s' % (self.__get_klass_name(node.getparent()), node.attrib['name']) self.__node_cache[name] = node for inc in gir_root.findall('./core:include', namespaces = self.__nsmap): inc_name = inc.attrib["name"] inc_version = inc.attrib["version"] gir_file = self.__find_gir_file('%s-%s.gir' % (inc_name, inc_version)) if not gir_file: warn('missing-gir-include', "Couldn't find a gir for %s-%s.gir" % (inc_name, inc_version)) continue if gir_file in self.__parsed_girs: continue self.__parsed_girs.add(gir_file) inc_gir_root = etree.parse(gir_file).getroot() self.__cache_nodes(inc_gir_root)
def __init__(self, name, generated, project_name, extension_name, source_file=None, ast=None, output_path='', raw_contents=None, comment=None, meta=None, pre_sorted=False, symbol_names=None): assert name if not generated: assert source_file is not None self.name = name basename = os.path.basename(name) name = os.path.splitext(basename)[0] ref = os.path.join(output_path, re.sub(r'\W+', '-', os.path.splitext(basename)[0])) pagename = '%s.html' % ref self.generated = generated self.project_name = project_name self.extension_name = extension_name self.source_file = source_file self.ast = ast self.raw_contents = raw_contents self.comment = comment self.pre_sorted = pre_sorted self.symbol_names = OrderedSet(symbol_names or []) self.output_attrs = None self.subpages = OrderedSet() self.symbols = [] self.private_symbols = [] self.typed_symbols = OrderedDict() self.by_parent_symbols = OrderedDict() self.formatted_contents = None self.detailed_description = None self.build_path = None self.cached_paths = OrderedSet() if comment: meta = comment.meta elif meta: meta = meta else: meta = {} self.meta = {} for key, value in meta.items(): try: self.meta.update(Schema(Page.meta_schema).validate({ key.replace('_', '-').lower(): value})) except SchemaError as err: warn('invalid-page-metadata', '%s: Invalid metadata: \n%s, discarding metadata' % (self.name, str(err))) if not self.meta.get('extra'): self.meta['extra'] = defaultdict() self.title = self.meta.get( 'title', cmark.title_from_ast(self.ast) if ast else '') self.thumbnail = self.meta.get('thumbnail') self.short_description = self.meta.get('short-description', None) self.render_subpages = self.meta.get('render-subpages', True) self.link = Link(pagename, self.title or name, ref)