def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(
diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
else:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
Page.formatting_signal(self, formatter)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.write_page(self, output)
def __check_since_markers(self, app):
with open(self.__symbols_database) as f:
prev_symbols = set(json.load(f))
all_symbols = app.database.get_all_symbols()
inherited_sinces = set()
missing_since_syms = set()
for name, sym in all_symbols.items():
if name in prev_symbols:
continue
if not self.__check_has_since(sym):
missing_since_syms.add(sym)
else:
self.__add_children_with_since(inherited_sinces, sym)
for sym in missing_since_syms - inherited_sinces:
if sym.comment and sym.comment.filename:
filename = sym.comment.filename
lineno = sym.comment.endlineno - 1
else:
filename = sym.filename
lineno = sym.lineno
warn('missing-since-marker',
message="Missing since marker for %s" % sym.unique_name,
filename=filename,
lineno=lineno,
)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(formatter.get_output_folder(self),
self.link.ref)
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.cache_page(self)
def __update_meta(self, meta):
for key, value in meta.items():
try:
self.meta.update(
Schema(Page.meta_schema).validate(
{key.replace('_', '-').lower(): value}))
except SchemaError as _:
warn(
'invalid-page-metadata',
'%s: Invalid metadata: \n%s, discarding metadata' %
(self.source_file, str(_)))
if not self.meta.get('extra'):
self.meta['extra'] = defaultdict()
self.title = meta.get('title', self.title)
self.thumbnail = meta.get('thumbnail')
self.listed_symbols = OrderedSet(
meta.get('symbols') or self.symbol_names)
self.private_symbols = OrderedSet(
meta.get('private-symbols') or self.private_symbols)
self.symbol_names = OrderedSet(
meta.get('symbols') or self.symbol_names)
self.short_description = meta.get('short-description',
self.short_description)
self.render_subpages = meta.get('render-subpages',
self.render_subpages)
def __lookup_asset(self, asset, project, page):
src = asset.attrib.get('src')
if not src:
warn('no-image-src', 'Empty image source in %s' % page.source_file)
return
comps = urllib.parse.urlparse(src)
if comps.scheme:
return
folders = [os.path.dirname(page.source_file)] if not page.generated \
else []
folders += [
os.path.join(folder, os.path.pardir)
for folder in project.extra_asset_folders
]
for folder in folders:
path = os.path.abspath(os.path.join(folder, src))
if os.path.exists(path):
output_folder = os.path.join(self.get_output_folder(page),
os.path.dirname(page.link.ref))
project.extra_assets[os.path.join(output_folder, src)] = path
asset.attrib['src'] = os.path.join(output_folder, src)
return
warn('bad-image-src',
('In %s, a local assets refers to an unknown source (%s). '
'It should be available in one of these locations: %s') %
(page.source_file, src, str(folders)))
def page_from_raw_text(self, source_file, contents, extension_name):
"""
Banana banana
"""
raw_contents = contents
meta = {}
if contents.startswith('---\n'):
split = contents.split('\n...\n', 1)
if len(split) == 2:
contents = split[1]
try:
blocks = yaml.load_all(split[0], Loader=yaml.FullLoader)
for block in blocks:
if block:
meta.update(block)
except ConstructorError as exception:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (source_file,
str(exception)))
output_path = os.path.dirname(os.path.relpath(
source_file, next(iter(self.project.include_paths))))
ast = cmark.hotdoc_to_ast(contents, self, source_file)
return Page(source_file, False, self.project.sanitized_name, extension_name,
source_file=source_file, ast=ast, meta=meta, raw_contents=raw_contents,
output_path=output_path)
def page_from_raw_text(self, source_file, contents):
"""
Banana banana
"""
raw_contents = contents
meta = {}
if contents.startswith('---\n'):
split = contents.split('\n...\n', 1)
if len(split) == 2:
contents = split[1]
try:
blocks = yaml.load_all(split[0])
for block in blocks:
if block:
meta.update(block)
except ConstructorError as exception:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (source_file,
str(exception)))
output_path = os.path.dirname(os.path.relpath(
source_file, iter(self.__include_paths).next()))
ast = cmark.hotdoc_to_ast(contents, self)
return Page(source_file, ast, output_path, meta=meta,
raw_contents=raw_contents)
def warn(self, code, message):
"""
Shortcut function for `loggable.warn`
Args:
code: see `utils.loggable.warn`
message: see `utils.loggable.warn`
"""
warn(code, message)
def write_page(self, page, output):
root = etree.HTML(unicode(page.detailed_description))
id_nodes = {n.attrib['id']: "".join([x for x in n.itertext()])
for n in root.xpath('.//*[@id]')}
section_numbers = self.__init_section_numbers(root)
targets = root.xpath(
'.//*[self::h1 or self::h2 or self::h3 or '
'self::h4 or self::h5 or self::img]')
for target in targets:
section_number = self.__update_section_number(
target, section_numbers)
if 'id' in target.attrib:
continue
if target.tag == 'img':
text = target.attrib.get('alt')
else:
text = "".join([x for x in target.itertext()])
if not text:
continue
id_ = id_from_text(text)
ref_id = id_
index = 1
while id_ in id_nodes:
id_ = '%s%s' % (ref_id, index)
index += 1
if section_number:
target.text = '%s %s' % (section_number, target.text or '')
target.attrib['id'] = id_
id_nodes[id_] = text
empty_links = root.xpath('.//a[not(text()) and not(*)]')
for link in empty_links:
href = link.attrib.get('href')
if href and href.startswith('#'):
title = id_nodes.get(href.strip('#'))
if title:
link.text = title
else:
warn('bad-local-link',
"Empty anchor link to %s in %s points nowhere" %
(href, page.source_file))
link.text = "FIXME broken link to %s" % href
page.detailed_description = lxml.html.tostring(
root, doctype="<!DOCTYPE html>", encoding='unicode',
include_meta_content_type=True)
return Formatter.write_page(self, page, output)
def write_page(self, page, output):
root = etree.HTML(unicode(page.detailed_description))
id_nodes = {n.attrib['id']: "".join([x for x in n.itertext()])
for n in root.xpath('.//*[@id]')}
section_numbers = self.__init_section_numbers(root)
targets = root.xpath(
'.//*[self::h1 or self::h2 or self::h3 or '
'self::h4 or self::h5 or self::img]')
for target in targets:
section_number = self.__update_section_number(
target, section_numbers)
if 'id' in target.attrib:
continue
if target.tag == 'img':
text = target.attrib.get('alt')
else:
text = "".join([x for x in target.itertext()])
if not text:
continue
id_ = id_from_text(text)
ref_id = id_
index = 1
while id_ in id_nodes:
id_ = '%s%s' % (ref_id, index)
index += 1
if section_number:
target.text = '%s %s' % (section_number, target.text or '')
target.attrib['id'] = id_
id_nodes[id_] = text
empty_links = root.xpath('.//a[not(text()) and not(*)]')
for link in empty_links:
href = link.attrib.get('href')
if href and href.startswith('#'):
title = id_nodes.get(href.strip('#'))
if title:
link.text = title
else:
warn('bad-local-link',
"Empty anchor link to %s in %s points nowhere" %
(href, page.source_file))
link.text = "FIXME broken link to %s" % href
page.detailed_description = lxml.html.tostring(
root, doctype="<!DOCTYPE html>", encoding='unicode',
include_meta_content_type=True)
return Formatter.write_page(self, page, output)
def warn(self, code, message):
"""
Shortcut function for `loggable.warn`
Args:
code: see `utils.loggable.warn`
message: see `utils.loggable.warn`
"""
warn(code, message)
def get_clang_headers():
version = subprocess.check_output(['llvm-config', '--version']).strip()
prefix = subprocess.check_output(['llvm-config', '--prefix']).strip()
for lib in ['lib', 'lib64']:
p = os.path.join(prefix, lib, 'clang', version, 'include')
if os.path.exists(p):
return p
warn('clang-headers-not-found', CLANG_HEADERS_WARNING)
def __parse_yaml_comment(self, comment, filename):
res = {}
try:
blocks = yaml.load_all(comment.raw_comment, Loader=yaml.SafeLoader)
for block in blocks:
if block:
res.update(block)
except (ConstructorError, yaml.parser.ParserError) as exception:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (filename, str(exception)))
return res
def __dummy_system_message(instance, level, message, *children, **kwargs):
if comment.lineno != -1 and 'line' in kwargs:
lineno = comment.lineno + kwargs.get('line')
else:
lineno = -1
warn(
'python-doc-issue',
message=message,
filename=comment.filename,
lineno=lineno)
def get_clang_headers():
version = subprocess.check_output(['llvm-config',
'--version']).strip().decode()
prefix = subprocess.check_output(['llvm-config',
'--prefix']).strip().decode()
for lib in ['lib', 'lib64']:
p = os.path.join(prefix, lib, 'clang', version, 'include')
if os.path.exists(p):
return p
warn('clang-headers-not-found', CLANG_HEADERS_WARNING)
def __format_content(self, formatter, link_resolver):
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(
diag.code,
message=diag.message,
filename=self.source_file or self.name)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
def __init__(self,
source_file,
ast,
output_path,
project_name,
meta=None,
raw_contents=None):
"Banana banana"
assert source_file
basename = os.path.basename(source_file)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.pre_sorted = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.project_name = project_name
self.cached_paths = OrderedSet()
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.render_subpages = meta.get('render-subpages', True)
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, ref)
def _formatting_page_cb(self, formatter: Formatter, page: Page) -> None:
''' Replace Meson refman tags
Links of the form [[function]] are automatically replaced
with valid links to the correct URL. To reference objects / types use the
[[@object]] syntax.
'''
link_regex = re.compile(
r'(\[\[#?@?([ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\.)*[ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\]\])(.)?',
re.MULTILINE)
for m in link_regex.finditer(page.formatted_contents):
i = m.group(1)
obj_id: str = i[2:-2]
obj_id = re.sub(r'[ \n\t]', '', obj_id) # Remove whitespaces
# Marked as inside a code block?
in_code_block = False
if obj_id.startswith('#'):
in_code_block = True
obj_id = obj_id[1:]
if obj_id not in self._data:
warn(
'unknown-refman-link',
f'{Path(page.name).name}: Unknown Meson refman link: "{obj_id}"'
)
continue
# Just replaces [[!file.id]] paths with the page file (no fancy HTML)
if obj_id.startswith('!'):
page.formatted_contents = page.formatted_contents.replace(
i, self._data[obj_id])
continue
# Fancy links for functions and methods
text = obj_id
if text.startswith('@'):
text = text[1:]
elif in_code_block:
if m.group(3) != '(':
text = text + '()'
else:
text = text + '()'
if not in_code_block:
text = f'<code>{text}</code>'
link = f'<a href="{self._data[obj_id]}"><ins>{text}</ins></a>'
page.formatted_contents = page.formatted_contents.replace(
i, link, 1)
def __formatting_page_cb(self, formatter, page):
# Only allow editing markdown pages
if page.generated:
return
root = self.__get_repo_root(page)
if not root:
warn('source-not-in-git-repo',
'File %s not in a git repository' % (page.source_file))
edit_link = self.__repo + '/edit/' + self.__branch + \
'/' + os.path.relpath(page.source_file, root)
page.output_attrs['html']['edit_button'] = \
'<a href=%s data-hotdoc-role="edit-button">' \
'Edit on github</a>' % edit_link
def __update_links(self, page, doc_root, id_nodes):
rel_path = os.path.join(self.get_output_folder(page), page.link.ref)
links = doc_root.xpath('.//*[@data-hotdoc-role="main"]//a')
for link in links:
href = link.attrib.get('href')
if href and href.startswith('#'):
if not link.text and not link.getchildren():
id_node = id_nodes.get(href.strip('#'))
if id_node is not None:
link.text = ''.join([x for x in id_node.itertext()])
else:
warn('bad-local-link',
"Empty anchor link to %s in %s points nowhere" %
(href, page.name))
link.text = "FIXME broken link to %s" % href
link.attrib["href"] = rel_path + href
def create_symbol(self, type_, **kwargs):
"""
Banana banana
"""
unique_name = kwargs.get('unique_name')
if not unique_name:
unique_name = kwargs.get('display_name')
kwargs['unique_name'] = unique_name
filename = kwargs.get('filename')
if filename:
filename = os.path.abspath(filename)
kwargs['filename'] = os.path.abspath(filename)
if unique_name in self.__symbols and not type_ == ProxySymbol:
warn('symbol-redefined', "%s(unique_name=%s, filename=%s, project=%s)"
" has already been defined: %s" % (type_.__name__, unique_name, filename,
kwargs.get('project_name'),
self.get_symbol(unique_name)))
return None
aliases = kwargs.pop('aliases', [])
alias_symbols = []
for alias in aliases:
alias_symbols.append(
self.create_symbol(ProxySymbol,
unique_name=alias,
target=unique_name))
symbol = type_()
debug('Created symbol with unique name %s' % unique_name,
'symbols')
for key, value in list(kwargs.items()):
setattr(symbol, key, value)
symbol.aliases += alias_symbols
if not isinstance(symbol, ProxySymbol):
self.__symbols[unique_name] = symbol
self.__aliased[unique_name].extend(aliases)
for alias in self.__aliased[unique_name]:
self.__aliases[alias] = symbol
return symbol
def __redirect_if_needed(self, formatter, link_resolver):
redirect = self.meta.get("redirect")
if not redirect:
return False
link = link_resolver.get_named_link(redirect)
if link:
if formatter.extension.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
self_dir = os.path.dirname(page_path)
if self_dir == self.link.ref:
self_dir = ""
self.meta["redirect"] = os.path.relpath(link.ref, self_dir)
else:
warn('markdown-bad-link', "Bad redirect link '%s' in page: %s"
% (redirect, self.name))
return True
def __include_file_cb(self, include_path, line_ranges, symbol_name):
if not include_path.endswith(".c") or not symbol_name:
return None
if not line_ranges:
line_ranges = [(1, -1)]
symbol = self.app.database.get_symbol(symbol_name)
if symbol and symbol.filename != include_path:
symbol = None
if not symbol:
scanner = ClangScanner(self.app, self.project, self)
scanner.scan([include_path], self.flags, self.app.incremental,
True, ['*.c', '*.h'])
symbol = self.app.database.get_symbol(symbol_name)
if not symbol:
warn(
'bad-c-inclusion',
"Trying to include symbol %s but could not be found in "
"%s" % (symbol_name, include_path))
return None
res = ''
for n, (start, end) in enumerate(line_ranges):
if n != 0:
res += "\n...\n"
start += symbol.extent_start - 2
if end > 0:
end += (symbol.extent_start - 1) # We are inclusive here
else:
end = symbol.extent_end
with open(include_path, "r") as _:
res += "\n".join(_.read().split("\n")[start:end])
if res:
return res, 'c'
return None
def __download_theme(self, uri):
sha = urllib.parse.parse_qs(uri.query).get('sha256')
cachedir = appdirs.user_cache_dir("hotdoc", "hotdoc")
os.makedirs(cachedir, exist_ok=True)
if sha:
sha = sha[0]
if sha and os.path.isdir(os.path.join(cachedir, "themes", sha)):
return os.path.join(cachedir, "themes", sha)
print("Downloading %s" % uri.geturl())
tarball = os.path.join(cachedir, "themes", os.path.basename(uri.path))
os.makedirs(os.path.dirname(tarball), exist_ok=True)
try:
urlretrieve(uri.geturl(), tarball, _download_progress_cb)
except (TimeoutError, urllib.error.HTTPError) as exce:
warn(
'download-theme-error',
"Error downloading %s: %s - Using default them" %
(tarball, exce))
return 'default'
if not sha:
with open(tarball) as file_:
sha = hashlib.sha256().update(file_.read()).hexdigest()
themepath = os.path.join(cachedir, "themes", sha)
try:
os.makedirs(os.path.join(themepath))
with tarfile.open(tarball) as file_:
file_.extractall(themepath)
except tarfile.ReadError:
warn(
'download-theme-error',
"%s is not a supported tarball - Using default theme" %
themepath)
os.rmdir(themepath)
return 'default'
return themepath
def __formatting_page_cb(self, formatter, page):
prism_theme = Formatter.theme_meta.get('prism-theme', 'prism')
prism_theme_path = '%s.css' % os.path.join(HERE, 'prism', 'themes',
prism_theme)
if os.path.exists(prism_theme_path):
page.output_attrs['html']['stylesheets'].add(prism_theme_path)
else:
warn('syntax-invalid-theme',
'Prism has no theme named %s' % prism_theme)
page.output_attrs['html']['scripts'].add(
os.path.join(HERE, 'prism', 'components', 'prism-core.js'))
page.output_attrs['html']['scripts'].add(
os.path.join(HERE, 'prism', 'plugins', 'autoloader',
'prism-autoloader.js'))
page.output_attrs['html']['scripts'].add(
os.path.join(HERE, 'prism_autoloader_path_override.js'))
folder = os.path.join('html', 'assets', 'prism_components')
self.__asset_folders.add(folder)
def __init__(self, source_file, ast, meta=None, raw_contents=None):
"Banana banana"
assert source_file
if os.path.isabs(source_file):
basename = os.path.basename(source_file)
else:
basename = source_file.replace('/', '-')
name = os.path.splitext(basename)[0]
pagename = '%s.html' % name
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file,
str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, name)
def __include_file_cb(self, include_path, line_ranges, symbol_name):
if not include_path.endswith(".c") or not symbol_name:
return None
if not line_ranges:
line_ranges = [(1, -1)]
symbol = self.doc_repo.doc_database.get_symbol(symbol_name)
if symbol and symbol.filename != include_path:
symbol = None
if not symbol:
scanner = ClangScanner(self.doc_repo, self)
scanner.scan([include_path], CExtension.flags,
self.doc_repo.incremental, True, ['*.c', '*.h'])
symbol = self.doc_repo.doc_database.get_symbol(symbol_name)
if not symbol:
warn('bad-c-inclusion',
"Trying to include symbol %s but could not be found in "
"%s" % (symbol_name, include_path))
return None
res = ''
for n, (start, end) in enumerate(line_ranges):
if n != 0:
res += "\n...\n"
start += symbol.extent_start - 2
if end > 0:
end += (symbol.extent_start - 1) # We are inclusive here
else:
end = symbol.extent_end
with open(include_path, "r") as _:
res += "\n".join(_.read().split("\n")[start:end])
if res:
return res, 'c'
return None
def format(self, formatter, link_resolver, root, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(
os.path.relpath(formatter.get_output_folder(),
'html').lstrip('./'),
self.link.ref)
Link.relativize_link_signal.connect(self.__relativize_link_cb)
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(
diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
else:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
Page.formatting_signal(self, formatter)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
Link.relativize_link_signal.disconnect(self.__relativize_link_cb)
if output:
formatter.write_page(self, root, output)
def __parse_tag(self, name, desc):
if name.lower() == "since":
return self.__parse_since_tag(name, desc)
if name.lower() == "returns":
return self.__parse_returns_tag(name, desc)
if name.lower() == "return value":
return self.__parse_returns_tag("returns", desc)
if name.lower() == "stability":
return self.__parse_stability_tag("stability", desc)
if name.lower() == "deprecated":
return self.__parse_deprecated_tag("deprecated", desc)
if name.lower() == "topic":
return self.__parse_topic_tag("topic", desc)
validator = self.project.tag_validators.get(name)
if not validator:
warn('gtk-doc', "FIXME no tag validator")
return None
if not validator.validate(desc):
warn('gtk-doc', "invalid value for tag %s : %s" % (name, desc))
return None
return Tag(name=name, description=desc, value=desc)
def __init__(self, source_file, ast, meta=None, raw_contents=None):
"Banana banana"
assert source_file
if os.path.isabs(source_file):
basename = os.path.basename(source_file)
else:
basename = source_file.replace('/', '-')
name = os.path.splitext(basename)[0]
pagename = '%s.html' % name
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, name)
def get_link(self, link_resolver):
"""
Banana banana
"""
res = link_resolver.resolving_link_signal(self)
if not res:
ref = self.ref
else:
ref = res[0] or self.ref
if self.__mandatory and not self.__warned:
if not ref:
warn('mandatory-link-not-found',
'Mandatory link %s could not be resolved' % self)
self.__warned = True
if not res:
return ref, None
if res[1]:
return ref, dict_to_html_attrs(res[1])
return ref, None
def page_from_raw_text(self, source_file, contents):
"""
Banana banana
"""
raw_contents = contents
meta = {}
if contents.startswith('---\n'):
split = contents.split('\n...\n', 1)
if len(split) == 2:
contents = split[1]
try:
blocks = yaml.load_all(split[0])
for block in blocks:
if block:
meta.update(block)
except ConstructorError as exception:
warn(
'invalid-page-metadata', '%s: Invalid metadata: \n%s' %
(source_file, str(exception)))
ast = cmark.hotdoc_to_ast(contents, self)
return Page(source_file, ast, meta=meta, raw_contents=raw_contents)
def __clone_and_update_repo(self):
cachedir = appdirs.user_cache_dir("hotdoc", "hotdoc")
repo_url, repo_path = _split_repo_url(self.__repository)
if repo_url is None or repo_path is None:
warn(
"git-error", "%s doesn't seem to contain a repository URL" %
(self.__repository))
return None
sanitize = str.maketrans('/@:', '___')
repo = os.path.join(cachedir, 'git-upload',
repo_url.translate(sanitize))
try:
call(['git', 'rev-parse', '--show-toplevel'],
cwd=repo,
stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError, FileNotFoundError):
print("cloning %s" % repo_url)
try:
subprocess.check_call(['git', 'clone', repo_url, repo])
except subprocess.CalledProcessError as exc:
warn(
"git-error", "Could not clone %s in %s: %s" %
(repo_url, repo, exc.output))
return None
try:
call(['git', 'checkout', self.__remote_branch[1]],
cwd=repo,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
warn(
"git-error", "Could not checkout branch %s in %s: %s" %
(self.__remote_branch[1], repo, exc.output))
return None
try:
call(['git', 'pull'], cwd=repo, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
warn(
"git-error", "Could not update %s (in %s) from remote %s" %
(repo_url, repo, exc.output))
return None
return os.path.join(repo, repo_path)
def cache_nodes(gir_root, all_girs):
'''
Identify and store all the gir symbols the symbols we will document
may link to, or be typed with
'''
ns_node = gir_root.find('./{%s}namespace' % NS_MAP['core'])
id_prefixes = ns_node.attrib['{%s}identifier-prefixes' % NS_MAP['c']]
sym_prefixes = ns_node.attrib['{%s}symbol-prefixes' % NS_MAP['c']]
id_key = '{%s}identifier' % NS_MAP['c']
for node in gir_root.xpath('.//*[@c:identifier]', namespaces=NS_MAP):
make_translations(node.attrib[id_key], node)
id_type = c_ns('type')
glib_type = glib_ns('type-name')
class_tag = core_ns('class')
callback_tag = core_ns('callback')
interface_tag = core_ns('interface')
for node in gir_root.xpath(
'.//*[not(self::core:type) and not (self::core:array)][@c:type or @glib:type-name]',
namespaces=NS_MAP):
try:
name = node.attrib[id_type]
except KeyError:
name = node.attrib[glib_type]
make_translations(name, node)
gi_name = '.'.join(get_gi_name_components(node))
ALL_GI_TYPES[gi_name] = get_klass_name(node)
if node.tag in (class_tag, interface_tag):
__update_hierarchies(ns_node.attrib.get('name'), node, gi_name)
make_translations('%s::%s' % (name, name), node)
__generate_smart_filters(id_prefixes, sym_prefixes, node)
elif node.tag in (callback_tag, ):
ALL_CALLBACK_TYPES.add(node.attrib[c_ns('type')])
for field in gir_root.xpath('.//self::core:field', namespaces=NS_MAP):
unique_name = get_field_c_name(field)
make_translations(unique_name, field)
for node in gir_root.xpath('.//core:property', namespaces=NS_MAP):
name = '%s:%s' % (get_klass_name(
node.getparent()), node.attrib['name'])
make_translations(name, node)
for node in gir_root.xpath('.//glib:signal', namespaces=NS_MAP):
name = '%s::%s' % (get_klass_name(
node.getparent()), node.attrib['name'])
make_translations(name, node)
for node in gir_root.xpath('.//core:virtual-method', namespaces=NS_MAP):
name = get_symbol_names(node)[0]
make_translations(name, node)
for inc in gir_root.findall('./core:include', namespaces=NS_MAP):
inc_name = inc.attrib["name"]
inc_version = inc.attrib["version"]
gir_file = __find_gir_file('%s-%s.gir' % (inc_name, inc_version),
all_girs)
if not gir_file:
warn('missing-gir-include',
"Couldn't find a gir for %s-%s.gir" % (inc_name, inc_version))
continue
if gir_file in __PARSED_GIRS:
continue
__PARSED_GIRS.add(gir_file)
inc_gir_root = etree.parse(gir_file).getroot()
cache_nodes(inc_gir_root, all_girs)
def write_page(self, page, build_root, output):
root = etree.HTML(unicode(page.detailed_description))
id_nodes = {n.attrib['id']: "".join([x for x in n.itertext()])
for n in root.xpath('.//*[@id]')}
section_numbers = self.__init_section_numbers(root)
targets = root.xpath(
'.//*[self::h1 or self::h2 or self::h3 or '
'self::h4 or self::h5 or self::img]')
for target in targets:
section_number = self.__update_section_number(
target, section_numbers)
if 'id' in target.attrib:
continue
if target.tag == 'img':
text = target.attrib.get('alt')
else:
text = "".join([x for x in target.itertext()])
if not text:
continue
id_ = id_from_text(text)
ref_id = id_
index = 1
while id_ in id_nodes:
id_ = '%s%s' % (ref_id, index)
index += 1
if section_number:
target.text = '%s %s' % (section_number, target.text or '')
target.attrib['id'] = id_
id_nodes[id_] = text
empty_links = root.xpath('.//a[not(text()) and not(*)]')
for link in empty_links:
href = link.attrib.get('href')
if href and href.startswith('#'):
title = id_nodes.get(href.strip('#'))
if title:
link.text = title
else:
warn('bad-local-link',
"Empty anchor link to %s in %s points nowhere" %
(href, page.source_file))
link.text = "FIXME broken link to %s" % href
page.detailed_description = lxml.html.tostring(
root, doctype="<!DOCTYPE html>", encoding='unicode',
include_meta_content_type=True)
full_path = Formatter.write_page(self, page, build_root, output)
images = root.xpath('.//img')
# All required assets should now be in place
for img in images:
src = img.attrib.get('src')
if not src:
warn('no-image-src',
'Empty image source in %s' % page.source_file)
continue
comps = urlparse.urlparse(src)
if comps.scheme:
continue
path = os.path.abspath(os.path.join(
os.path.dirname(full_path), src))
if not os.path.exists(path):
warn('bad-image-src',
('In %s, a local image refers to an unknown source (%s). '
'It should be available in the build folder, at %s') %
(page.source_file, src, path))
continue
return full_path
def scan(self,
filenames,
options,
full_scan,
full_scan_patterns,
fail_fast=False,
all_sources=None):
if all_sources is None:
self.__all_sources = []
else:
self.__all_sources = all_sources
index = cindex.Index.create()
flags = cindex.TranslationUnit.PARSE_INCOMPLETE | cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
info('scanning %d C source files' % len(filenames))
self.filenames = filenames
# FIXME: er maybe don't do that ?
args = ["-Wno-attributes"]
args.append("-isystem%s" % CLANG_HEADERS)
args.extend(options)
self.symbols = {}
self.parsed = set({})
debug('CFLAGS %s' % ' '.join(args))
header_guarded = set()
for filename in self.filenames:
if filename in self.parsed:
continue
do_full_scan = any(
fnmatch(filename, p) for p in full_scan_patterns)
if do_full_scan:
debug('scanning %s' % filename)
tu = index.parse(filename, args=args, options=flags)
for diag in tu.diagnostics:
s = diag.format()
warn('clang-diagnostic', 'Clang issue : %s' % str(diag))
self.__parse_file(filename, tu, full_scan)
if (cindex.conf.lib.clang_isFileMultipleIncludeGuarded(
tu, tu.get_file(filename))):
header_guarded.add(filename)
for include in tu.get_includes():
fname = os.path.abspath(str(include.include))
if (cindex.conf.lib.clang_isFileMultipleIncludeGuarded(
tu, tu.get_file(fname))):
if fname in self.filenames:
header_guarded.add(fname)
self.__parse_file(fname, tu, full_scan)
if not full_scan:
comment_parser = GtkDocParser(self.project)
CCommentExtractor(self.__doc_db,
comment_parser).parse_comments(filenames)
return True
def parse_comment(self, comment, filename, lineno, endlineno,
include_paths=None, stripped=False):
"""
Returns a Comment given a string
"""
if not stripped and not self.__validate_c_comment(comment.strip()):
return None
comment = unicode(comment.decode('utf8'))
title_offset = 0
column_offset = 0
raw_comment = comment
if not stripped:
try:
while comment[column_offset * -1 - 1] != '\n':
column_offset += 1
except IndexError:
column_offset = 0
comment, title_offset = self.__strip_comment(comment)
split = re.split(r'\n[\W]*\n', comment, maxsplit=1)
try:
block_name, parameters, annotations = \
self.__parse_title_and_parameters(split[0])
except HotdocSourceException as _:
warn('gtk-doc-bad-syntax',
message=_.message,
filename=filename,
lineno=lineno + title_offset)
return None
params_offset = 0
for param in parameters:
param.filename = filename
param.lineno = lineno
param_offset = param.line_offset
param.line_offset = title_offset + params_offset + 1
params_offset += param_offset
param.col_offset = column_offset
if not block_name:
return None
description_offset = 0
description = ""
tags = []
if len(split) > 1:
n_lines = len(comment.split('\n'))
description_offset = (title_offset + n_lines -
len(split[1].split('\n')))
description, tags = self.__parse_description_and_tags(split[1])
title = None
short_description = None
actual_parameters = {}
for param in parameters:
if param.name.lower() == 'short_description':
short_description = param
elif param.name.lower() == 'title':
title = param
else:
actual_parameters[param.name] = param
annotations = {annotation.name: annotation for annotation in
annotations}
tags = {tag.name.lower(): tag for tag in tags}
block = Comment(name=block_name, filename=filename, lineno=lineno,
endlineno=endlineno,
annotations=annotations, params=actual_parameters,
description=description,
short_description=short_description,
title=title, tags=tags, raw_comment=raw_comment)
block.line_offset = description_offset
block.col_offset = column_offset
return block
def __formatted_cb(self, app):
repo = self.__local_repo
if not repo:
if not self.__repository or not self.__activate:
return
repo = self.__clone_and_update_repo()
if not repo:
return
try:
call(['git', 'rev-parse', '--show-toplevel'],
cwd=repo,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exception:
warn(
'no-local-repository',
"Specified local repository %s does not exist, "
"not uploading. (%s)" % (repo, str(exception)))
if subprocess.call(['git', 'diff-index', '--quiet', 'HEAD'],
cwd=repo) != 0:
warn(
'dirty-local-repository',
"Specified local repository %s is dirty, not uploading." %
(repo))
return
html_path = os.path.join(app.output, "html/")
print("Uploading %s to %s!" % (html_path, repo))
print("Removing previous assets %s" % os.path.join(repo, "assets/"))
shutil.rmtree(os.path.join(repo, "assets/"), ignore_errors=True)
print("Removing previous html files")
# pylint: disable=unused-variable
for root, subdirs, files in os.walk(repo):
for filename in files:
if filename.endswith('.html'):
os.remove(os.path.join(root, filename))
print("Copying newly built files")
for filename in os.listdir(html_path):
built_f = os.path.join(html_path, filename)
copy_dest = os.path.join(repo, filename)
recursive_overwrite(built_f, copy_dest)
call(['git', 'add', filename], cwd=repo)
if self.__copy_only:
return
if subprocess.call(['git', 'diff-index', '--quiet', 'HEAD'],
cwd=repo) == 0:
print("No changes to push")
return
print("Committing %s" % self.__commit_message)
subprocess.check_call(
['git', 'commit', '-a', '-m', self.__commit_message], cwd=repo)
print("Pushing to %s" % '/'.join(self.__remote_branch))
subprocess.check_call(['git', 'push'] + self.__remote_branch, cwd=repo)
def scan(self, filenames, options, incremental, full_scan,
full_scan_patterns, fail_fast=False):
index = clang.cindex.Index.create()
flags = clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
info('scanning %d C source files' % len(filenames))
self.filenames = filenames
# FIXME: er maybe don't do that ?
args = ["-Wno-attributes"]
args.append ("-isystem%s" % get_clang_headers())
args.extend (options)
self.symbols = {}
self.parsed = set({})
debug('CFLAGS %s' % ' '.join(args))
header_guarded = set()
for filename in self.filenames:
if filename in self.parsed:
continue
do_full_scan = any(fnmatch(filename, p) for p in full_scan_patterns)
if do_full_scan:
debug('scanning %s' % filename)
tu = index.parse(filename, args=args, options=flags)
for diag in tu.diagnostics:
warn('clang-diagnostic', 'Clang issue : %s' % str(diag))
self.__parse_file (filename, tu, full_scan)
if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(filename))):
header_guarded.add(filename)
for include in tu.get_includes():
fname = os.path.abspath(str(include.include))
if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(fname))):
if fname in self.filenames:
header_guarded.add(fname)
self.__parse_file (fname, tu, full_scan)
if not full_scan:
for filename in filenames:
with open (filename, 'r') as f:
skip_next_symbol = filename in header_guarded
debug('Getting comments in %s' % filename)
cs = get_comments (filename)
for c in cs:
if c[4]:
block = self.__raw_comment_parser.parse_comment(c[0],
c[1], c[2], c[3], self.doc_repo.include_paths)
if block is not None:
self.doc_repo.doc_database.add_comment(block)
elif not skip_next_symbol:
if filename.endswith('.h'):
self.__create_macro_from_raw_text(c)
else:
skip_next_symbol = False
return True
def comment_to_ast(self, comment, link_resolver):
"""
Given a gtk-doc comment string, returns an opaque PyCapsule
containing the document root.
This is an optimization allowing to parse the docstring only
once, and to render it multiple times with
`ast_to_html`, links discovery and
most of the link resolution being lazily done in that second phase.
If you don't care about performance, you should simply
use `translate`.
Args:
text: unicode, the docstring to parse.
link_resolver: hotdoc.core.links.LinkResolver, an object
which will be called to retrieve `hotdoc.core.links.Link`
objects.
Returns:
capsule: A PyCapsule wrapping an opaque C pointer, which
can be passed to `ast_to_html`
afterwards.
diagnostics: A list of diagnostics as output by the gtk-doc cmark
extension
"""
assert comment is not None
text = comment.description
if (GtkDocStringFormatter.remove_xml_tags or comment.filename in
GtkDocStringFormatter.gdbus_codegen_sources):
text = re.sub('<.*?>', '', text)
if GtkDocStringFormatter.escape_html:
text = cgi.escape(text)
ast, diagnostics = cmark.gtkdoc_to_ast(text, link_resolver)
for diag in diagnostics:
if (comment.filename and comment.filename not in
GtkDocStringFormatter.gdbus_codegen_sources):
column = diag.column + comment.col_offset
if diag.lineno == 0:
column += comment.initial_col_offset
lines = text.split('\n')
line = lines[diag.lineno]
i = 0
while line[i] == ' ':
i += 1
column += i - 1
if diag.lineno > 0 and any([c != ' ' for c in
lines[diag.lineno - 1]]):
column += 1
lineno = -1
if comment.lineno != -1:
lineno = (comment.lineno - 1 + comment.line_offset +
diag.lineno)
warn(
diag.code,
message=diag.message,
filename=comment.filename,
lineno=lineno,
column=column)
return ast
def __cache_nodes(self, gir_root):
ns_node = gir_root.find('./{%s}namespace' % self.__nsmap['core'])
id_prefixes = ns_node.attrib['{%s}identifier-prefixes' % self.__nsmap['c']]
sym_prefixes = ns_node.attrib['{%s}symbol-prefixes' % self.__nsmap['c']]
id_key = '{%s}identifier' % self.__nsmap['c']
for node in gir_root.xpath(
'.//*[@c:identifier]',
namespaces=self.__nsmap):
self.__node_cache[node.attrib[id_key]] = node
id_type = '{%s}type' % self.__nsmap['c']
class_tag = '{%s}class' % self.__nsmap['core']
interface_tag = '{%s}interface' % self.__nsmap['core']
for node in gir_root.xpath(
'.//*[not(self::core:type) and not (self::core:array)][@c:type]',
namespaces=self.__nsmap):
name = node.attrib[id_type]
self.__node_cache[name] = node
if node.tag in [class_tag, interface_tag]:
gi_name = '.'.join(self.__get_gi_name_components(node))
self.__class_nodes[gi_name] = node
get_type_function = node.attrib.get('{%s}get-type' %
self.__nsmap['glib'])
self.__get_type_functions.add(get_type_function)
self.__node_cache['%s::%s' % (name, name)] = node
self.__generate_smart_filters(id_prefixes, sym_prefixes, node)
for node in gir_root.xpath(
'.//core:property',
namespaces=self.__nsmap):
name = '%s:%s' % (self.__get_klass_name(node.getparent()),
node.attrib['name'])
self.__node_cache[name] = node
for node in gir_root.xpath(
'.//glib:signal',
namespaces=self.__nsmap):
name = '%s::%s' % (self.__get_klass_name(node.getparent()),
node.attrib['name'])
self.__node_cache[name] = node
for node in gir_root.xpath(
'.//core:virtual-method',
namespaces=self.__nsmap):
name = '%s:::%s' % (self.__get_klass_name(node.getparent()),
node.attrib['name'])
self.__node_cache[name] = node
for inc in gir_root.findall('./core:include',
namespaces = self.__nsmap):
inc_name = inc.attrib["name"]
inc_version = inc.attrib["version"]
gir_file = self.__find_gir_file('%s-%s.gir' % (inc_name,
inc_version))
if not gir_file:
warn('missing-gir-include', "Couldn't find a gir for %s-%s.gir" %
(inc_name, inc_version))
continue
if gir_file in self.__parsed_girs:
continue
self.__parsed_girs.add(gir_file)
inc_gir_root = etree.parse(gir_file).getroot()
self.__cache_nodes(inc_gir_root)
def __init__(self, name, generated, project_name, extension_name,
source_file=None,
ast=None,
output_path='',
raw_contents=None,
comment=None,
meta=None,
pre_sorted=False,
symbol_names=None):
assert name
if not generated:
assert source_file is not None
self.name = name
basename = os.path.basename(name)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-', os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.generated = generated
self.project_name = project_name
self.extension_name = extension_name
self.source_file = source_file
self.ast = ast
self.raw_contents = raw_contents
self.comment = comment
self.pre_sorted = pre_sorted
self.symbol_names = OrderedSet(symbol_names or [])
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.private_symbols = []
self.typed_symbols = OrderedDict()
self.by_parent_symbols = OrderedDict()
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.cached_paths = OrderedSet()
if comment:
meta = comment.meta
elif meta:
meta = meta
else:
meta = {}
self.meta = {}
for key, value in meta.items():
try:
self.meta.update(Schema(Page.meta_schema).validate({
key.replace('_', '-').lower(): value}))
except SchemaError as err:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s, discarding metadata' %
(self.name, str(err)))
if not self.meta.get('extra'):
self.meta['extra'] = defaultdict()
self.title = self.meta.get(
'title', cmark.title_from_ast(self.ast) if ast else '')
self.thumbnail = self.meta.get('thumbnail')
self.short_description = self.meta.get('short-description', None)
self.render_subpages = self.meta.get('render-subpages', True)
self.link = Link(pagename, self.title or name, ref)
