def get_dependencies(self):
"""
Retrieve the set of all dependencies for a given configuration.
Returns:
utils.utils.OrderedSet: The set of all dependencies for the
tracked configuration.
"""
all_deps = OrderedSet()
for key, _ in self.__config.items():
if key in self.__cli:
continue
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1])
for key, _ in self.__cli.items():
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1])
if self.__conf_file is not None:
all_deps.add(self.__conf_file)
all_deps.add(self.get_path("sitemap", rel_to_cwd=True))
cwd = os.getcwd()
return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def get_stale_files(self, all_files, fileset_name):
"""
Banana banana
"""
stale = OrderedSet()
previous_mtimes = self.mtimes[fileset_name]
new_mtimes = defaultdict()
for filename in all_files:
mtime = get_mtime(filename)
prev_mtime = previous_mtimes.pop(filename, None)
new_mtimes[filename] = mtime
if mtime == prev_mtime:
continue
stale.add(filename)
self.mtimes[fileset_name] = new_mtimes
unlisted = set(previous_mtimes.keys())
ChangeTracker.all_stale_files |= stale
ChangeTracker.all_unlisted_files |= unlisted
return stale, unlisted
def get_dependencies(self):
"""
Retrieve the set of all dependencies for a given configuration.
Returns:
utils.utils.OrderedSet: The set of all dependencies for the
tracked configuration.
"""
all_deps = OrderedSet()
for key, _ in list(self.__config.items()):
if key in self.__cli:
continue
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1 - 1])
for key, _ in list(self.__cli.items()):
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1 - 1])
if self.conf_file is not None:
all_deps.add(self.conf_file)
all_deps.add(self.get_path("sitemap", rel_to_cwd=True))
cwd = os.getcwd()
return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def get_stale_files(self, all_files, fileset_name):
"""
Banana banana
"""
stale = OrderedSet()
previous_mtimes = self.mtimes[fileset_name]
new_mtimes = defaultdict()
for filename in all_files:
mtime = get_mtime(filename)
prev_mtime = previous_mtimes.pop(filename, None)
new_mtimes[filename] = mtime
if mtime == prev_mtime:
continue
stale.add(filename)
self.mtimes[fileset_name] = new_mtimes
unlisted = set(previous_mtimes.keys())
ChangeTracker.all_stale_files |= stale
ChangeTracker.all_unlisted_files |= unlisted
return stale, unlisted
def __resolve_patterns(self, source_patterns, from_conf):
if source_patterns is None:
return OrderedSet()
all_files = OrderedSet()
for item in source_patterns:
item = self.__abspath(item, from_conf)
if '*' in item:
all_files |= glob.glob(item)
else:
all_files.add(item)
return all_files
def get_markdown_files(self, dir_):
"""
Get all the markdown files in a folder, recursively
Args:
dir_: str, a toplevel folder to walk.
"""
md_files = OrderedSet()
for root, _, files in os.walk(dir_):
for name in files:
split = os.path.splitext(name)
if len(split) == 1:
continue
if split[1] in ('.markdown', '.md', '.yaml'):
md_files.add(os.path.join(root, name))
return md_files
def get_markdown_files(self, dir_):
"""
Get all the markdown files in a folder, recursively
Args:
dir_: str, a toplevel folder to walk.
"""
md_files = OrderedSet()
for root, _, files in os.walk(dir_):
for name in files:
split = os.path.splitext(name)
if len(split) == 1:
continue
if split[1] in ('.markdown', '.md', '.yaml'):
md_files.add(os.path.join(root, name))
return md_files
def __resolve_patterns(self, source_patterns, from_conf):
if source_patterns is None:
return OrderedSet()
all_files = OrderedSet()
for item in source_patterns:
item = self.__abspath(item, from_conf)
if item in all_files:
continue
if not os.path.exists(item):
all_files |= glob.glob(item)
else:
all_files.add(item)
return all_files
class Page(object):
"Banana banana"
meta_schema = {
Optional('title'): And(unicode, len),
Optional('symbols'): Schema([And(unicode, len)]),
Optional('short-description'): And(unicode, len)
}
resolving_symbol_signal = Signal()
formatting_signal = Signal()
def __init__(self, source_file, ast, meta=None, raw_contents=None):
"Banana banana"
assert source_file
if os.path.isabs(source_file):
basename = os.path.basename(source_file)
else:
basename = source_file.replace('/', '-')
name = os.path.splitext(basename)[0]
pagename = '%s.html' % name
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, name)
def __getstate__(self):
return {
'ast': None,
'title': self.title,
'raw_contents': self.raw_contents,
'short_description': self.short_description,
'extension_name': self.extension_name,
'link': self.link,
'meta': self.meta,
'source_file': self.source_file,
'comment': self.comment,
'generated': self.generated,
'is_stale': False,
'formatted_contents': None,
'detailed_description': None,
'output_attrs': None,
'symbols': [],
'typed_symbols': {},
'subpages': self.subpages,
'symbol_names': self.symbol_names
}
def resolve_symbols(self, doc_database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `doc_database`, and added to lists of actual symbols, sorted
by symbol class.
"""
typed_symbols_list = namedtuple('TypedSymbolsList',
['name', 'symbols'])
self.typed_symbols[Symbol] = typed_symbols_list('FIXME symbols', [])
self.typed_symbols[FunctionSymbol] = typed_symbols_list(
"Functions", [])
self.typed_symbols[CallbackSymbol] = typed_symbols_list(
"Callback Functions", [])
self.typed_symbols[FunctionMacroSymbol] = typed_symbols_list(
"Function Macros", [])
self.typed_symbols[ConstantSymbol] = typed_symbols_list(
"Constants", [])
self.typed_symbols[ExportedVariableSymbol] = typed_symbols_list(
"Exported Variables", [])
self.typed_symbols[StructSymbol] = typed_symbols_list(
"Data Structures", [])
self.typed_symbols[EnumSymbol] = typed_symbols_list("Enumerations", [])
self.typed_symbols[AliasSymbol] = typed_symbols_list("Aliases", [])
self.typed_symbols[SignalSymbol] = typed_symbols_list("Signals", [])
self.typed_symbols[PropertySymbol] = typed_symbols_list(
"Properties", [])
self.typed_symbols[VFunctionSymbol] = typed_symbols_list(
"Virtual Methods", [])
self.typed_symbols[ClassSymbol] = typed_symbols_list("Classes", [])
self.typed_symbols[InterfaceSymbol] = typed_symbols_list(
"Interfaces", [])
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = doc_database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, link_resolver,
doc_database)
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver)
self.symbol_names.add(sym.unique_name)
class_syms = self.typed_symbols[ClassSymbol].symbols
interface_syms = self.typed_symbols[InterfaceSymbol].symbols
struct_syms = self.typed_symbols[StructSymbol].symbols
if self.title is None:
if class_syms:
self.title = class_syms[0].display_name
elif interface_syms:
self.title = interface_syms[0].display_name
elif struct_syms:
self.title = struct_syms[0].display_name
if self.comment is None:
if class_syms and class_syms[0].comment:
self.comment = class_syms[0].comment
elif interface_syms and interface_syms[0].comment:
self.comment = interface_syms[0].comment
elif struct_syms and struct_syms[0].comment:
self.comment = struct_syms[0].comment
# pylint: disable=no-self-use
def __fetch_comment(self, sym, doc_database):
old_comment = sym.comment
new_comment = doc_database.get_comment(sym.unique_name)
sym.comment = Comment(sym.unique_name)
if new_comment:
sym.comment = new_comment
elif old_comment:
if not old_comment.filename in (ChangeTracker.all_stale_files |
ChangeTracker.all_unlisted_files):
sym.comment = old_comment
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(self.comment.title,
link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1>%s</h1>' % self.title
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
else:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
Page.formatting_signal(self, formatter)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.write_page(self, output)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug(
'Formatting symbol %s in page %s' %
(symbol.unique_name, self.source_file), 'formatting')
symbol.skip = not formatter.format_symbol(symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, link_resolver,
doc_database):
if sym:
self.__fetch_comment(sym, doc_database)
new_symbols = sum(Page.resolving_symbol_signal(self, sym), [])
all_syms.add(sym)
for symbol in new_symbols:
self.__query_extra_symbols(symbol, all_syms, link_resolver,
doc_database)
def __resolve_symbol(self, symbol, link_resolver):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (self.link.ref, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (self.link.ref, link.id_)
tsl = self.typed_symbols[type(symbol)]
tsl.symbols.append(symbol)
self.symbols.append(symbol)
debug(
'Resolved symbol %s to page %s' %
(symbol.display_name, self.link.ref), 'resolution')
class TestDocTree(unittest.TestCase):
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(os.path.join(
here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(
here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(
here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(
here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def tearDown(self):
self.__remove_tmp_dirs()
del self.test_ext
del self.core_ext
def get_generated_doc_folder(self):
return os.path.join(self.__priv_dir, 'generated')
def get_base_doc_folder(self):
return self.__md_dir
def get_private_folder(self):
return self.__priv_dir
def __parse_sitemap(self, text):
path = os.path.join(self.__md_dir, 'sitemap.txt')
with io.open(path, 'w', encoding='utf-8') as _:
_.write(text)
return self.sitemap_parser.parse(path)
def __create_md_file(self, name, contents):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
_.write(contents)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
def __create_src_file(self, name, symbols):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
for symbol in symbols:
_.write('%s\n' % symbol)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
return path
def __remove_src_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __remove_md_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __touch_src_file(self, name):
path = os.path.join(self.__md_dir, name)
touch(path)
def __remove_tmp_dirs(self):
shutil.rmtree(self.__md_dir, ignore_errors=True)
shutil.rmtree(self.__priv_dir, ignore_errors=True)
shutil.rmtree(self.__src_dir, ignore_errors=True)
shutil.rmtree(self.__output_dir, ignore_errors=True)
shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)
def test_basic(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
# We do not care about ordering
self.assertSetEqual(
set(pages.keys()),
set([u'index.markdown',
u'section.markdown']))
index = pages.get('index.markdown')
self.assertEqual(index.title, u'My documentation')
def test_basic_incremental(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Building from scratch, all pages are stale
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set([u'index.markdown',
u'section.markdown']))
doc_tree.persist()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Nothing changed, no page is stale
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set({}))
# But we still have our pages
self.assertSetEqual(
set(doc_tree.get_pages()),
set([u'index.markdown',
u'section.markdown']))
touch(os.path.join(self.__md_dir, u'section.markdown'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set([u'section.markdown']))
def __assert_extension_names(self, doc_tree, name_map):
pages = doc_tree.get_pages()
for name, ext_name in name_map.items():
page = pages[name]
self.assertEqual(ext_name, page.extension_name)
def __assert_stale(self, doc_tree, expected_stale):
stale_pages = doc_tree.get_stale_pages()
for pagename in expected_stale:
self.assertIn(pagename, stale_pages)
stale_pages.pop(pagename)
self.assertEqual(len(stale_pages), 0)
def __create_test_layout(self):
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
sources = []
sources.append(self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2']))
sources.append(self.__create_src_file(
'source_b.test',
['symbol_3',
'symbol_4']))
self.test_ext.index = 'test-index.markdown'
self.test_ext.sources = sources
self.test_ext.setup()
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'core_page.markdown',
(u'# My non-extension page\n'))
self.__create_md_file(
'test-index.markdown',
(u'# My test index\n'))
self.__create_md_file(
'test-section.markdown',
(u'# My test section\n'
'\n'
'Linking to [a generated page](source_a.test)\n'))
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__create_md_file(
'page_y.markdown',
(u'# Page Y\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree, sitemap
def __update_test_layout(self, doc_tree, sitemap):
self.test_ext.reset()
self.test_ext.setup()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree
def test_extension_basic(self):
doc_tree, _ = self.__create_test_layout()
self.__assert_extension_names(
doc_tree,
{u'index.markdown': 'core',
u'test-index': 'test-extension',
u'test-section.markdown': 'test-extension',
u'source_a.test': 'test-extension',
u'source_b.test': 'test-extension',
u'page_x.markdown': 'test-extension',
u'page_y.markdown': 'test-extension',
u'core_page.markdown': 'core'})
all_pages = doc_tree.get_pages()
self.assertEqual(len(all_pages), 8)
self.__assert_stale(doc_tree, all_pages)
self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
self.assertIn('source_a.test',
all_pages['test-section.markdown'].subpages)
def test_extension_override(self):
self.__create_md_file(
'source_a.test.markdown',
(u'# My override\n'))
doc_tree, _ = self.__create_test_layout()
page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
self.assertEqual(
os.path.basename(page.source_file),
'source_a.test.markdown')
self.assertEqual(
cmark.ast_to_html(page.ast, None),
u'<h1>My override</h1>\n')
def test_parse_yaml(self):
inp = (u'index.markdown\n')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'---\n'
'title: A random title\n'
'symbols: [symbol_1, symbol_2]\n'
'...\n'
'# My documentation\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
page = pages.get('index.markdown')
self.assertEqual(
cmark.ast_to_html(page.ast, None),
u'<h1>My documentation</h1>\n')
self.assertEqual(page.title, u'A random title')
self.assertEqual(
page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
def test_empty_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html">My documentation</a></p>\n')
def test_labeled_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[a label](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html">a label</a></p>\n')
def test_anchored_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown#subsection)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html#subsection">My documentation</a></p>\n')
# pylint: disable=too-many-statements
def test_extension_incremental(self):
doc_tree, sitemap = self.__create_test_layout()
doc_tree.persist()
# Here we touch source_a.test, as its symbols were
# all contained in a generated page, only that page
# should now be stale
self.__touch_src_file('source_a.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_a.test']))
doc_tree.persist()
# We now touch source_b.test, which symbols are contained
# both in a generated page and a user-provided one.
# We expect both pages to be stale
self.__touch_src_file('source_b.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
doc_tree.persist()
# This one is trickier: we unlist symbol_3 from
# page_x, which means the symbol should now be
# documented in the generated page for source_b.test.
# We expect both pages to be stale, and make sure
# they contain the right symbols
self.__create_md_file(
'page_x.markdown',
(u'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet())
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# Let's make sure the opposite use case works as well,
# we relocate symbol_3 in page_x , both page_x and
# the generated page for source_b.test should be stale
# and the symbols should be back to their original
# layout.
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
doc_tree.persist()
# We now move the definition of symbol_3 to source_a.test,
# we thus expect the generated page for source_a.test to be
# stale because its source changed, same for source_b.test,
# and page_x.markdown should be stale as well because the
# definition of symbol_3 may have changed. The final
# symbol layout should not have changed however.
self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2',
'symbol_3'])
self.__create_src_file(
'source_b.test',
['symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_a.test',
'source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
source_a_page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
doc_tree.persist()
# And we rollback again
self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2'])
self.__create_src_file(
'source_b.test',
['symbol_3',
'symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_a.test',
'source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
source_a_page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
doc_tree.persist()
# Now we'll try removing page_x altogether
self.__remove_md_file('page_x.markdown')
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
new_sitemap = self.__parse_sitemap(inp)
doc_tree = self.__update_test_layout(doc_tree, new_sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# And rollback again
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['page_x.markdown',
'source_b.test']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
doc_tree.persist()
class Extension(Configurable):
"""
All extensions should inherit from this base class
Attributes:
extension_name: str, the unique name of this extension, should
be overriden and namespaced appropriately.
project: project.Project, the Project instance which documentation
hotdoc is working on.
formatter: formatter.Formatter, may be subclassed.
argument_prefix (str): Short name of this extension, used as a prefix
to add to automatically generated command-line arguments.
"""
# pylint: disable=unused-argument
extension_name = "base-extension"
argument_prefix = ''
paths_arguments = {}
path_arguments = {}
written_out_sitemaps = set()
def __init__(self, app, project):
"""Constructor for `Extension`.
This should never get called directly.
Args:
project: The `project.Project` instance which documentation
is being generated.
"""
self.project = project
self.app = app
self.sources = set()
self.smart_sources = []
self.index = None
self.source_roots = OrderedSet()
self._created_symbols = DefaultOrderedDict(OrderedSet)
self.__package_root = None
self.__toplevel_comments = OrderedSet()
self.formatter = self._make_formatter()
# pylint: disable=no-self-use
def warn(self, code, message):
"""
Shortcut function for `loggable.warn`
Args:
code: see `utils.loggable.warn`
message: see `utils.loggable.warn`
"""
warn(code, message)
# pylint: disable=no-self-use
def error(self, code, message):
"""
Shortcut function for `utils.loggable.error`
Args:
code: see `utils.loggable.error`
message: see `utils.loggable.error`
"""
error(code, message)
def debug(self, message, domain=None):
"""
Shortcut function for `utils.loggable.debug`
Args:
message: see `utils.loggable.debug`
domain: see `utils.loggable.debug`
"""
if domain is None:
domain = self.extension_name
debug(message, domain)
def info(self, message, domain=None):
"""
Shortcut function for `utils.loggable.info`
Args:
message: see `utils.loggable.info`
domain: see `utils.loggable.info`
"""
if domain is None:
domain = self.extension_name
info(message, domain)
def reset(self):
"""
This function is only useful for testing purposes, at least
for now.
"""
self._created_symbols = DefaultOrderedDict(OrderedSet)
self.__package_root = None
def get_pagename(self, name):
self.__find_package_root()
# Find the longest prefix
longest = None
for path in OrderedSet([self.__package_root]) | self.source_roots:
commonprefix = os.path.commonprefix([path, name])
if commonprefix == path and (longest is None or len(path) > len(longest)):
longest = path
if longest is not None:
return os.path.relpath(name, longest)
return name
def get_symbol_page(self, symbol_name, symbol_pages, smart_pages, section_links):
if symbol_name in symbol_pages:
return symbol_pages[symbol_name]
symbol = self.app.database.get_symbol(symbol_name)
assert symbol is not None
if symbol.parent_name and symbol.parent_name != symbol_name:
page = self.get_symbol_page(
symbol.parent_name, symbol_pages, smart_pages, section_links)
else:
smart_key = self._get_smart_key(symbol)
if smart_key is None:
return None
if smart_key in smart_pages:
page = smart_pages[smart_key]
else:
pagename = self.get_pagename(smart_key)
page = Page(smart_key, True, self.project.sanitized_name, self.extension_name,
output_path=os.path.dirname(pagename))
if page.link.ref in section_links:
self.warn('output-page-conflict',
'Creating a page for symbol %s would overwrite the page '
'declared in a toplevel comment (%s)' % (symbol_name, page.link.ref))
page = None
else:
smart_pages[smart_key] = page
if page is not None:
symbol_pages[symbol_name] = page
return page
def _get_toplevel_comments(self):
return self.__toplevel_comments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
def make_pages(self):
# All symbol names that no longer need to be assigned to a page
dispatched_symbol_names = set()
# Map symbol names with pages
# This is used for assigning symbols with a parent to the page
# where their parent will be rendered, unless the symbol has been
# explicitly assigned or ignored
symbol_pages = {}
smart_pages = OrderedDict()
# Map pages with the sources they explicitly import
imported_sources = {}
# This is simply used as a conflict detection mechanism, see
# hotdoc.core.tests.test_doc_tree.TestTree.test_section_and_path_conflict
section_links = set()
# These are used as a duplicate detection mechanism, map
# relocated or ignored symbols to the source files where they were initially
# listed
relocated_symbols = {}
private_symbols = {}
# First we make one page per toplevel comment (eg. SECTION comment)
# This is the highest priority mechanism for sorting symbols
for comment in self._get_toplevel_comments():
# Programming error from extension author
assert comment.name
symbol_names = comment.meta.pop('symbols', [])
private_symbol_names = comment.meta.pop('private-symbols', [])
sources = comment.meta.pop('sources', None)
page = Page(comment.name, True, self.project.sanitized_name, self.extension_name,
comment=comment)
for symbol_name in symbol_names:
if symbol_name in relocated_symbols:
self.warn('symbol-listed-twice',
'Symbol %s listed in %s was already listed in %s' %
(symbol_name, comment.filename, relocated_symbols[symbol_name]))
continue
elif symbol_name in private_symbols:
self.warn('symbol-listed-twice',
'Symbol %s listed in %s was marked as private in %s' %
(symbol_name, comment.filename, private_symbols[symbol_name]))
continue
else:
page.symbol_names.add(symbol_name)
symbol_pages[symbol_name] = page
relocated_symbols[symbol_name] = comment.filename
dispatched_symbol_names.add(symbol_name)
for symbol_name in private_symbol_names:
if symbol_name in relocated_symbols:
self.warn('symbol-listed-twice',
'Symbol %s marked private in %s was already listed in %s' %
(symbol_name, comment.filename, relocated_symbols[symbol_name]))
continue
elif symbol_name in private_symbols:
self.warn('symbol-listed-twice',
'Symbol %s marked as private in %s was '
'already marked as private in %s' %
(symbol_name, comment.filename, private_symbols[symbol_name]))
continue
private_symbols[symbol_name] = comment.filename
symbol_pages[symbol_name] = None
dispatched_symbol_names.add(symbol_name)
section_links.add(page.link.ref)
smart_key = self._get_comment_smart_key(comment)
if smart_key in smart_pages:
smart_pages[comment.name] = page
else:
smart_pages[smart_key] = page
if sources is not None:
abs_sources = []
for source in sources:
if os.path.isabs(source):
abs_sources.append(source)
else:
abs_sources.append(os.path.abspath(os.path.join(
os.path.dirname(comment.filename), source)))
imported_sources[page] = abs_sources
# Used as a duplicate detection mechanism
relocated_sources = {}
# We now browse all the pages with explicitly imported sources
# Importing sources has a lower level of priority than importing
# symbols, which is why we do that in a separate loop
for page, sources in imported_sources.items():
for source in sources:
if source not in self._get_all_sources():
self.warn('invalid-relocated-source',
'Source %s does not exist but is relocated in %s' %
(source, page.name))
continue
if source in relocated_sources:
self.warn('invalid-relocated-source',
'Source %s relocated in %s was already relocated in %s' %
(source, page.name, relocated_sources[source]))
continue
if source in self._created_symbols:
symbol_names = OrderedSet(
self._created_symbols[source]) - dispatched_symbol_names
page.symbol_names |= symbol_names
dispatched_symbol_names |= symbol_names
relocated_sources[source] = page.name
# We now browse all the symbols we have created
for _, symbol_names in self._created_symbols.items():
for symbol_name in symbol_names:
if symbol_name in dispatched_symbol_names:
continue
page = self.get_symbol_page(
symbol_name, symbol_pages, smart_pages, section_links)
# Can be None if creating a page to hold the symbol conflicts with
# a page explicitly declared in a toplevel comment or a parent has been
# marked as private
if page is None:
continue
page.symbol_names.add(symbol_name)
dispatched_symbol_names.add(symbol_name)
# Finally we make our index page
if self.index:
index_page = self.project.tree.parse_page(
self.index, self.extension_name)
else:
index_page = Page('%s-index' % self.argument_prefix, True, self.project.sanitized_name,
self.extension_name)
if not index_page.title:
index_page.title = self._get_smart_index_title()
smart_pages['%s-index' % self.argument_prefix] = index_page
return smart_pages
def setup(self):
"""
Extension subclasses should implement this to scan whatever
source files they have to scan, and connect to the various
signals they have to connect to.
Note that this will be called *after* the `tree.Tree`
of this instance's `Extension.project` has been fully
constructed, but before its `tree.Tree.resolve_symbols`
method has been called.
"""
pass
@staticmethod
def get_dependencies():
"""
Override this to return the list of extensions this extension
depends on if needed.
Returns:
list: A list of `ExtDependency` instances.
"""
return []
def parse_toplevel_config(self, config):
"""Parses and make use of the toplevel configuration."""
self.formatter.parse_toplevel_config(config)
def parse_config(self, config):
"""
Override this, making sure to chain up first, if your extension adds
its own custom command line arguments, or you want to do any further
processing on the automatically added arguments.
The default implementation will set attributes on the extension:
- 'sources': a set of absolute paths to source files for this extension
- 'index': absolute path to the index for this extension
Additionally, it will set an attribute for each argument added with
`Extension.add_path_argument` or `Extension.add_paths_argument`, with
the extension's `Extension.argument_prefix` stripped, and dashes
changed to underscores.
Args:
config: a `config.Config` instance
"""
prefix = self.argument_prefix
self.sources = config.get_sources(prefix)
self.smart_sources = [
self._get_smart_filename(s) for s in self.sources]
self.index = config.get_index(prefix)
self.source_roots = OrderedSet(
config.get_paths('%s_source_roots' % prefix))
for arg, dest in list(self.paths_arguments.items()):
val = config.get_paths(arg)
setattr(self, dest, val)
for arg, dest in list(self.path_arguments.items()):
val = config.get_path(arg)
setattr(self, dest, val)
self.formatter.parse_config(config)
def add_attrs(self, symbol, **kwargs):
"""
Helper for setting symbol extension attributes
"""
for key, val in kwargs.items():
symbol.add_extension_attribute(self.extension_name, key, val)
def get_attr(self, symbol, attrname):
"""
Helper for getting symbol extension attributes
"""
return symbol.extension_attributes.get(self.extension_name, {}).get(
attrname, None)
@staticmethod
def add_arguments(parser):
"""
Subclasses may implement this method to add their own arguments to
the hotdoc binary.
In this function, you should add an argument group to the passed-in
parser, corresponding to your extension.
You can then add arguments to that argument group.
Example::
@staticmethod
def add_arguments(parser):
group = parser.add_argument_group('Chilidoc',
'Delicious Hotdoc extension')
Chilidoc.add_sources_argument(group)
group.add_argument('--chili-spicy', action='store_true',
help='Whether to add extra pepper')
Args:
parser (argparse.ArgumentParser): Main hotdoc argument parser
"""
pass
@classmethod
def get_assets_licensing(cls):
return {}
@classmethod
def add_index_argument(cls, group):
"""
Subclasses may call this to add an index argument.
Args:
group: arparse.ArgumentGroup, the extension argument group
prefix: str, arguments have to be namespaced
"""
prefix = cls.argument_prefix
group.add_argument(
'--%s-index' % prefix, action="store",
dest="%s_index" % prefix,
help=("Name of the %s root markdown file, can be None" % (
cls.extension_name)))
@classmethod
def add_sources_argument(cls, group, allow_filters=True, prefix=None, add_root_paths=False):
"""
Subclasses may call this to add sources and source_filters arguments.
Args:
group: arparse.ArgumentGroup, the extension argument group
allow_filters: bool, Whether the extension wishes to expose a
source_filters argument.
prefix: str, arguments have to be namespaced.
"""
prefix = prefix or cls.argument_prefix
group.add_argument("--%s-sources" % prefix,
action="store", nargs="+",
dest="%s_sources" % prefix.replace('-', '_'),
help="%s source files to parse" % prefix)
if allow_filters:
group.add_argument("--%s-source-filters" % prefix,
action="store", nargs="+",
dest="%s_source_filters" % prefix.replace(
'-', '_'),
help="%s source files to ignore" % prefix)
if add_root_paths:
group.add_argument("--%s-source-roots" % prefix,
action="store", nargs="+",
dest="%s_source_roots" % prefix.replace(
'-', '_'),
help="%s source root directories allowing files "
"to be referenced relatively to those" % prefix)
@classmethod
def add_path_argument(cls, group, argname, dest=None, help_=None):
"""
Subclasses may call this to expose a path argument.
Args:
group: arparse.ArgumentGroup, the extension argument group
argname: str, the name of the argument, will be namespaced.
dest: str, similar to the `dest` argument of
`argparse.ArgumentParser.add_argument`, will be namespaced.
help_: str, similar to the `help` argument of
`argparse.ArgumentParser.add_argument`.
"""
prefixed = '%s-%s' % (cls.argument_prefix, argname)
if dest is None:
dest = prefixed.replace('-', '_')
final_dest = dest[len(cls.argument_prefix) + 1:]
else:
final_dest = dest
dest = '%s_%s' % (cls.argument_prefix, dest)
group.add_argument('--%s' % prefixed, action='store',
dest=dest, help=help_)
cls.path_arguments[dest] = final_dest
@classmethod
def add_paths_argument(cls, group, argname, dest=None, help_=None):
"""
Subclasses may call this to expose a paths argument.
Args:
group: arparse.ArgumentGroup, the extension argument group
argname: str, the name of the argument, will be namespaced.
dest: str, similar to the `dest` argument of
`argparse.ArgumentParser.add_argument`, will be namespaced.
help_: str, similar to the `help` argument of
`argparse.ArgumentParser.add_argument`.
"""
prefixed = '%s-%s' % (cls.argument_prefix, argname)
if dest is None:
dest = prefixed.replace('-', '_')
final_dest = dest[len(cls.argument_prefix) + 1:]
else:
final_dest = dest
dest = '%s_%s' % (cls.argument_prefix, dest)
group.add_argument('--%s' % prefixed, action='store', nargs='+',
dest=dest, help=help_)
cls.paths_arguments[dest] = final_dest
def add_comment(self, comment):
if comment.toplevel:
self.__toplevel_comments.add(comment)
else:
self.app.database.add_comment(comment)
def create_symbol(self, *args, **kwargs):
"""
Extensions that discover and create instances of `symbols.Symbol`
should do this through this method, as it will keep an index
of these which can be used when generating a "naive index".
See `database.Database.create_symbol` for more
information.
Args:
args: see `database.Database.create_symbol`
kwargs: see `database.Database.create_symbol`
Returns:
symbols.Symbol: the created symbol, or `None`.
"""
if not kwargs.get('project_name'):
kwargs['project_name'] = self.project.project_name
sym = self.app.database.create_symbol(*args, **kwargs)
if sym:
# pylint: disable=unidiomatic-typecheck
if type(sym) != Symbol:
self._created_symbols[sym.filename].add(sym.unique_name)
return sym
def rename_symbol(self, unique_name, target):
sym = self.app.database.rename_symbol(unique_name, target)
# pylint: disable=unidiomatic-typecheck
if sym and type(sym) != Symbol:
self._created_symbols[sym.filename].remove(target)
self._created_symbols[sym.filename].add(sym.unique_name)
def _make_formatter(self):
return Formatter(self)
def get_possible_path(self, name):
self.__find_package_root()
for path in OrderedSet([self.__package_root]) | self.source_roots:
possible_path = os.path.join(path, name)
if possible_path in self._get_all_sources():
return self._get_smart_filename(possible_path)
return None
def __find_package_root(self):
if self.__package_root:
return
commonprefix = os.path.commonprefix(
list(self._get_all_sources()) + list(self.source_roots))
self.__package_root = os.path.dirname(commonprefix)
def _get_smart_index_title(self):
return 'Reference Manual'
def _get_all_sources(self):
return self.sources
def _get_smart_key(self, symbol):
return symbol.filename
def _get_smart_filename(self, filename):
return filename
def _get_comment_smart_key(self, comment):
return comment.filename
def format_page(self, page, link_resolver, output):
"""
Called by `project.Project.format_page`, to leave full control
to extensions over the formatting of the pages they are
responsible of.
Args:
page: tree.Page, the page to format.
link_resolver: links.LinkResolver, object responsible
for resolving links potentially mentioned in `page`
output: str, path to the output directory.
"""
debug('Formatting page %s' % page.link.ref, 'formatting')
if output:
actual_output = os.path.join(output,
'html')
if not os.path.exists(actual_output):
os.makedirs(actual_output)
else:
actual_output = None
page.format(self.formatter, link_resolver, actual_output)
def write_out_sitemap(self, opath):
"""
Banana banana
"""
if opath not in self.written_out_sitemaps:
Extension.formatted_sitemap = self.formatter.format_navigation(
self.app.project)
if Extension.formatted_sitemap:
escaped_sitemap = Extension.formatted_sitemap.replace(
'\\', '\\\\').replace('"', '\\"').replace('\n', '')
js_wrapper = 'sitemap_downloaded_cb("%s");' % escaped_sitemap
with open(opath, 'w') as _:
_.write(js_wrapper)
self.written_out_sitemaps.add(opath)
# pylint: disable=too-many-locals
def write_out_page(self, output, page):
"""
Banana banana
"""
subpages = OrderedDict({})
all_pages = self.project.tree.get_pages()
subpage_names = self.get_subpages_sorted(all_pages, page)
for pagename in subpage_names:
proj = self.project.subprojects.get(pagename)
if not proj:
cpage = all_pages[pagename]
sub_formatter = self.project.extensions[
cpage.extension_name].formatter
else:
cpage = proj.tree.root
sub_formatter = proj.extensions[cpage.extension_name].formatter
subpage_link, _ = cpage.link.get_link(self.app.link_resolver)
prefix = sub_formatter.get_output_folder(cpage)
if prefix:
subpage_link = '%s/%s' % (prefix, subpage_link)
subpages[subpage_link] = cpage
html_subpages = self.formatter.format_subpages(page, subpages)
js_dir = os.path.join(output, 'html', 'assets', 'js')
if not os.path.exists(js_dir):
os.makedirs(js_dir)
sm_path = os.path.join(js_dir, 'sitemap.js')
self.write_out_sitemap(sm_path)
self.formatter.write_out(page, html_subpages, output)
def get_subpages_sorted(self, pages, page):
"""Get @page subpages sorted appropriately."""
sorted_pages = []
to_sort = []
for subpage in page.subpages:
# Do not resort subprojects even if they are
# 'generated'.
if pages[subpage].pre_sorted:
sorted_pages.append(subpage)
else:
to_sort.append(subpage)
return sorted_pages + sorted(
to_sort, key=lambda p: pages[p].get_title().lower())
class GstExtension(Extension):
extension_name = 'gst-extension'
argument_prefix = 'gst'
__dual_links = {} # Maps myelement:XXX to GstMyElement:XXX
__parsed_cfiles = set()
__caches = {} # cachefile -> dict
__apps_sigs = set()
__all_plugins_symbols = set()
def __init__(self, app, project):
super().__init__(app, project)
self.cache = {}
self.c_sources = []
self.cache_file = None
self.plugin = None
self.__elements = {}
self.__raw_comment_parser = GtkDocParser(project,
section_file_matching=False)
self.__plugins = None
self.__toplevel_comments = OrderedSet()
self.list_plugins_page = None
# If we have a plugin with only one element, we render it on the plugin
# page.
self.unique_feature = None
self.__on_index_symbols = []
def _make_formatter(self):
return GstFormatter(self)
def create_symbol(self, *args, **kwargs):
sym = super().create_symbol(*args, **kwargs)
if self.unique_feature and sym:
self.__on_index_symbols.append(sym)
return sym
# pylint: disable=too-many-branches
def setup(self):
# Make sure the cache file is save when the whole project
# is done.
if self.cache_file not in GstExtension.__apps_sigs and self.cache_file:
GstExtension.__apps_sigs.add(self.cache_file)
if not self.cache_file:
if self.list_plugins_page:
self.__plugins = self.create_symbol(
GstPluginsSymbol,
display_name="All " +
self.project.project_name.replace("-", " ").title(),
unique_name=self.project.project_name + "-all-gst-plugins",
plugins=[],
all_plugins=True)
super().setup()
return
gather_gtk_doc_links()
comment_parser = GtkDocParser(self.project, False)
to_parse_sources = set(self.c_sources) - GstExtension.__parsed_cfiles
CCommentExtractor(self,
comment_parser).parse_comments(to_parse_sources)
GstExtension.__parsed_cfiles.update(self.c_sources)
if not self.cache:
error('setup-issue',
"No cache loaded or created for %s" % self.plugin)
plugins = []
if self.plugin:
pname = self.plugin
dot_idx = pname.rfind('.')
if dot_idx > 0:
pname = self.plugin[:dot_idx]
if pname.startswith('libgst'):
pname = pname[6:]
elif pname.startswith('gst'):
pname = pname[3:]
try:
plugin_node = {pname: self.cache[pname]}
except KeyError:
error('setup-issue', "Plugin %s not found" % pname)
else:
plugin_node = self.cache
for libfile, plugin in plugin_node.items():
plugin_sym = self.__parse_plugin(libfile, plugin)
if not plugin_sym:
continue
plugins.append(plugin_sym)
if not self.plugin:
self.__plugins = self.create_symbol(
GstPluginsSymbol,
display_name=self.project.project_name.replace("-",
" ").title(),
unique_name=self.project.project_name + "-gst-plugins",
plugins=plugins)
super().setup()
def _get_comment_smart_key(self, comment):
try:
return comment.title.description
except AttributeError:
return None
def _get_toplevel_comments(self):
if self.unique_feature:
return OrderedSet()
return self.__toplevel_comments
def get_plugin_comment(self):
if self.plugin:
res = self.app.database.get_comment("plugin-" + self.plugin)
return res
return None
def make_pages(self):
smart_pages = super().make_pages()
if not self.__plugins:
return None
if self.list_plugins_page is None:
index = smart_pages.get('gst-index')
if index is None:
return smart_pages
index.render_subpages = False
index.symbol_names.add(self.__plugins.unique_name)
for sym in self.__on_index_symbols:
index.symbol_names.add(sym.unique_name)
if self.unique_feature:
index.comment = self.app.database.get_comment(
"element-" + self.unique_feature)
else:
index.comment = self.get_plugin_comment()
return smart_pages
page = smart_pages.get(self.list_plugins_page)
page.render_subpages = False
page.extension_name = self.extension_name
page.symbol_names.add(self.__plugins.unique_name)
self.__plugins.plugins = self.__all_plugins_symbols
return smart_pages
def add_comment(self, comment):
# We handle toplevel comments ourself, make sure all comments
# end up in the database
comment.toplevel = False
super().add_comment(comment)
def _get_smart_index_title(self):
if self.plugin:
return self.__plugins.display_name
return 'GStreamer plugins documentation'
@staticmethod
def add_arguments(parser):
group = parser.add_argument_group('Gst extension', DESCRIPTION)
GstExtension.add_index_argument(group)
# GstExtension.add_order_generated_subpages(group)
GstExtension.add_sources_argument(group, prefix='gst-c')
group.add_argument('--gst-cache-file', default=None)
group.add_argument('--gst-list-plugins-page', default=None)
group.add_argument('--gst-plugin-name', default=None)
group.add_argument('--gst-plugins-path', default=None)
def parse_config(self, config):
self.c_sources = config.get_sources('gst_c')
self.cache_file = config.get('gst_cache_file')
self.plugin = config.get('gst_plugin_name')
self.list_plugins_page = config.get('gst_list_plugins_page', None)
info('Parsing config!')
self.cache = {}
if self.cache_file:
self.cache = GstExtension.__caches.get(self.cache_file)
if not self.cache:
try:
with open(self.cache_file) as _:
self.cache = json.load(_)
except FileNotFoundError:
pass
if self.cache is None:
self.cache = {}
GstExtension.__caches[self.cache_file] = self.cache
super().parse_config(config)
def _get_smart_key(self, symbol):
if self.unique_feature:
return None
if isinstance(symbol, GstPluginSymbol):
# PluginSymbol are rendered on the index page
return None
res = symbol.extra.get('gst-element-name')
if res:
res = res.replace("element-", "")
return res
# pylint: disable=too-many-locals
# pylint: disable=too-many-arguments
def __create_signal_symbol(self,
obj,
parent_uniquename,
name,
signal,
element_name,
parent_name=None):
atypes = signal['args']
instance_type = obj['hierarchy'][0]
unique_name = "%s::%s" % (parent_uniquename, name)
aliases = self._get_aliases(["%s::%s" % (instance_type, name)])
gi_extension = self.project.extensions.get('gi-extension')
python_lang = gi_extension.get_language('python')
args_type_names = []
args_type_names = [
(type_tokens_from_type_name('GstElement', python_lang), 'param_0')
]
for i, _ in enumerate(atypes):
args_type_names.append(
(type_tokens_from_type_name(atypes[i], python_lang),
'param_%s' % (i + 1)))
args_type_names.append(
(type_tokens_from_type_name('gpointer', python_lang), "udata"))
params = []
for comment_name in [unique_name] + aliases:
comment = self.app.database.get_comment(comment_name)
if comment:
for i, argname in enumerate(comment.params.keys()):
args_type_names[i] = (args_type_names[i][0], argname)
for tokens, argname in args_type_names:
params.append(ParameterSymbol(argname=argname, type_tokens=tokens))
type_name = signal['retval']
if type_name == 'void':
retval = [None]
else:
tokens = type_tokens_from_type_name(type_name, python_lang)
enum = signal.get('return-values')
if enum:
self.__create_enum_symbol(type_name,
enum,
obj.get('name', parent_uniquename),
parent_name=parent_name)
retval = [ReturnItemSymbol(type_tokens=tokens)]
return self.create_symbol(SignalSymbol,
parameters=params,
return_value=retval,
display_name=name,
unique_name=unique_name,
extra={'gst-element-name': element_name},
aliases=aliases,
parent_name=parent_name)
def __create_signal_symbols(self,
obj,
parent_uniquename,
element_name,
parent_name=None):
signals = obj.get('signals', {})
if not signals:
return
for name, signal in signals.items():
self.__create_signal_symbol(obj,
parent_uniquename,
name,
signal,
element_name,
parent_name=parent_name)
def __create_property_symbols(self,
obj,
parent_uniquename,
pagename,
parent_name=None):
properties = obj.get('properties', [])
if not properties:
return
gi_extension = self.project.extensions.get('gi-extension')
python_lang = gi_extension.get_language('python')
for name, prop in properties.items():
unique_name = '%s:%s' % (obj.get('name', parent_uniquename), name)
flags = [ReadableFlag()]
if prop['writable']:
flags += [WritableFlag()]
if prop['construct-only']:
flags += [ConstructOnlyFlag()]
elif prop['construct']:
flags += [ConstructFlag()]
type_name = prop['type-name']
tokens = type_tokens_from_type_name(type_name, python_lang)
type_ = QualifiedSymbol(type_tokens=tokens)
default = prop.get('default')
enum = prop.get('values')
if enum:
type_ = self.__create_enum_symbol(prop['type-name'],
enum,
obj.get(
'name',
parent_uniquename),
parent_name=parent_name)
if obj['hierarchy'][0] != parent_uniquename:
aliases = self._get_aliases(
['%s:%s' % (obj['hierarchy'][0], name)])
else:
aliases = []
res = self.app.database.get_symbol(unique_name)
if res is None:
res = self.create_symbol(
PropertySymbol,
prop_type=type_,
display_name=name,
unique_name=unique_name,
aliases=aliases,
parent_name=parent_name,
extra={'gst-element-name': pagename},
)
assert res
if not self.app.database.get_comment(unique_name):
comment = Comment(unique_name,
Comment(name=name),
description=prop['blurb'])
self.app.database.add_comment(comment)
# FIXME This is incorrect, it's not yet format time (from gi_extension)
extra_content = self.formatter.format_flags(flags)
res.extension_contents['Flags'] = extra_content
if default:
if prop['type-name'] in ['GstCaps', 'GstStructure']:
default = '<pre class="language-yaml">' + \
'<code class="language-yaml">%s</code></pre>' % default
res.extension_contents['Default value'] = default
def __create_enum_symbol(self,
type_name,
enum,
element_name,
parent_name=None):
display_name = re.sub(r"([a-z])([A-Z])", r"\g<1>-\g<2>",
type_name.replace('Gst', ''))
unique_name = type_name
if self.app.database.get_symbol(unique_name):
unique_name = element_name + '_' + type_name
symbol = self.app.database.get_symbol(unique_name)
if not symbol:
members = []
for val in enum:
value_unique_name = "%s::%s" % (type_name, val['name'])
if self.app.database.get_symbol(value_unique_name):
value_unique_name = "%s::%s" % (element_name + '_' +
type_name, val['name'])
member_sym = self.create_symbol(
GstNamedConstantValue,
unique_name=value_unique_name,
display_name=val['name'],
value=val['value'],
parent_name=parent_name,
val=val,
extra={'gst-element-name': element_name})
if member_sym:
members.append(member_sym)
symbol = self.create_symbol(
GstNamedConstantsSymbols,
anonymous=False,
raw_text=None,
display_name=display_name.capitalize(),
unique_name=unique_name,
parent_name=parent_name,
members=members,
extra={'gst-element-name': 'element-' + element_name})
elif not isinstance(symbol, GstNamedConstantsSymbols):
self.warn(
'symbol-redefined',
"EnumMemberSymbol(unique_name=%s, project=%s)"
" has already been defined: %s" %
(unique_name, self.project.project_name, symbol))
return None
if not symbol:
return None
symbol.values = enum
return symbol
def __create_object_type(self, pagename, _object):
if not _object:
return None
unique_name = _object['hierarchy'][0]
if self.app.database.get_symbol(unique_name):
return None
self.__create_property_symbols(_object,
unique_name,
pagename,
parent_name=unique_name)
self.__create_signal_symbols(_object,
unique_name,
pagename,
parent_name=unique_name)
return self.create_symbol(ClassSymbol,
hierarchy=create_hierarchy(_object),
display_name=unique_name,
unique_name=unique_name,
parent_name=unique_name,
extra={'gst-element-name': pagename})
def __create_pad_template_symbols(self, element, plugin_name):
templates = element.get('pad-templates', {})
if not templates:
return
for tname, template in templates.items():
name = tname.replace("%%", "%")
unique_name = '%s!%s' % (element['hierarchy'][0], name)
pagename = 'element-' + element['name']
object_type = self.__create_object_type(
pagename, template.get("object-type"))
self.create_symbol(GstPadTemplateSymbol,
name=name,
direction=template["direction"],
presence=template["presence"],
caps=template["caps"],
filename=plugin_name,
parent_name=None,
object_type=object_type,
display_name=name,
unique_name=unique_name,
extra={'gst-element-name': pagename})
def _get_aliases(self, aliases):
return [
alias for alias in aliases
if not self.app.database.get_symbol(alias)
]
def __extract_feature_comment(self, feature_type, feature):
pagename = feature_type + '-' + feature['name']
possible_comment_names = [pagename, feature['name']]
if feature_type == 'element':
possible_comment_names.append(feature['hierarchy'][0])
comment = None
for comment_name in possible_comment_names:
comment = self.app.database.get_comment(comment_name)
if comment:
break
description = feature.get('description')
if not comment:
comment = Comment(
pagename,
Comment(description=feature['name']),
description=description,
short_description=Comment(description=description))
self.app.database.add_comment(comment)
elif not comment.short_description:
comment.short_description = Comment(description=description)
comment.title = Comment(description=feature['name'])
comment.name = feature.get('name', pagename)
comment.meta['title'] = feature['name']
self.__toplevel_comments.add(comment)
return pagename, comment
def __parse_plugin(self, plugin_name, plugin):
elements = []
feature_names = list(plugin.get('elements', {}).keys()) \
+ list(plugin.get('tracers', {}).keys()) \
+ list(plugin.get('device-providers', {}).keys())
if self.plugin and len(feature_names) == 1:
self.unique_feature = feature_names[0]
for ename, tracer in plugin.get('tracers', {}).items():
tracer['name'] = ename
self.__extract_feature_comment("tracer", tracer)
for provider_name, provider in plugin.get('device-providers',
{}).items():
provider['name'] = provider_name
_, comment = self.__extract_feature_comment("provider", provider)
comment.description += """\n\n# Provided device example"""
self.__create_object_type(provider_name,
provider.get('device-example'))
for ename, element in plugin.get('elements', {}).items():
element['name'] = ename
pagename, _ = self.__extract_feature_comment("element", element)
aliases = self._get_aliases([pagename, element['hierarchy'][0]])
sym = self.create_symbol(GstElementSymbol,
parent_name=None,
display_name=element['name'],
hierarchy=create_hierarchy(element),
unique_name=element['name'],
filename=plugin_name,
extra={'gst-element-name': pagename},
rank=str(element['rank']),
author=element['author'],
classification=element['klass'],
plugin=plugin['filename'],
aliases=aliases,
package=plugin['package'])
if not sym:
continue
self.__elements[element['name']] = sym
self.__create_property_symbols(element, element['name'], pagename)
self.__create_signal_symbols(element, element['name'], pagename)
self.__create_pad_template_symbols(element, plugin_name)
elements.append(sym)
plugin = self.create_symbol(
GstPluginSymbol,
description=plugin['description'],
display_name=plugin_name,
unique_name='plugin-' + plugin['filename'],
license=plugin['license'],
package=plugin['package'],
filename=plugin['filename'],
elements=elements,
extra={'gst-plugins': 'plugins-' + plugin['filename']})
if not plugin:
return None
self.__all_plugins_symbols.add(plugin)
if self.plugin:
self.__plugins = plugin
return plugin
def __get_link_cb(self, resolver, name):
link = self.__dual_links.get(name)
if link:
# Upsert link on the first run
if isinstance(link, str):
sym = self.app.database.get_symbol(link)
link = sym.link
if link:
return link
return (resolver, name)
def format_page(self, page, link_resolver, output):
link_resolver.get_link_signal.connect(GIExtension.search_online_links)
super().format_page(page, link_resolver, output)
link_resolver.get_link_signal.disconnect(
GIExtension.search_online_links)
class Page:
"Banana banana"
meta_schema = {Optional('title'): And(str, len),
Optional('short-description'): And(str),
Optional('description'): And(str),
Optional('render-subpages'): bool,
Optional('auto-sort'): bool,
Optional('full-width'): bool,
Optional('see-also'): And(str, len),
Optional('extra'): Schema({str: object}),
Optional('thumbnail'): And(str, len),
Optional('include'): And(str, len),
Optional('redirect'): str,
}
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
def __init__(self, name, generated, project_name, extension_name,
source_file=None,
ast=None,
output_path='',
raw_contents=None,
comment=None,
meta=None,
pre_sorted=False,
symbol_names=None):
assert name
if not generated:
assert source_file is not None
self.name = name
basename = os.path.basename(name)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-', os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.generated = generated
self.project_name = project_name
self.extension_name = extension_name
self.source_file = source_file
self.ast = ast
self.raw_contents = raw_contents
self.comment = comment
self.pre_sorted = pre_sorted
self.symbol_names = OrderedSet(symbol_names or [])
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.private_symbols = []
self.typed_symbols = OrderedDict()
self.by_parent_symbols = OrderedDict()
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.cached_paths = OrderedSet()
if comment:
meta = comment.meta
elif meta:
meta = meta
else:
meta = {}
self.meta = {}
for key, value in meta.items():
try:
self.meta.update(Schema(Page.meta_schema).validate({
key.replace('_', '-').lower(): value}))
except SchemaError as err:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s, discarding metadata' %
(self.name, str(err)))
if not self.meta.get('extra'):
self.meta['extra'] = defaultdict()
self.title = self.meta.get(
'title', cmark.title_from_ast(self.ast) if ast else '')
self.thumbnail = self.meta.get('thumbnail')
self.short_description = self.meta.get('short-description', None)
self.render_subpages = self.meta.get('render-subpages', True)
self.link = Link(pagename, self.title or name, ref)
def __repr__(self):
return "<Page %s>" % self.name
@staticmethod
def __get_empty_typed_symbols():
typed_symbols_list = namedtuple(
'TypedSymbolsList', ['name', 'symbols'])
empty_typed_symbols = {}
for subclass in all_subclasses(Symbol):
empty_typed_symbols[subclass] = typed_symbols_list(
subclass.get_plural_name(), [])
return empty_typed_symbols
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
self.typed_symbols = self.__get_empty_typed_symbols()
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(
sym, all_syms, tree, link_resolver, database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
if self.meta.get("auto-sort", True):
all_syms = sorted(all_syms, key=lambda x: x.unique_name)
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
# Always put symbols with no parent at the end
no_parent_syms = self.by_parent_symbols.pop(None, None)
if no_parent_syms:
self.by_parent_symbols[None] = no_parent_syms
for sym_type in [ClassSymbol, AliasSymbol, InterfaceSymbol,
StructSymbol]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.name)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
# pylint: disable=no-self-use
def __fetch_comment(self, sym, database):
sym.comment = database.get_comment(
sym.unique_name) or Comment(sym.unique_name)
for sym in sym.get_children_symbols():
if isinstance(sym, Symbol):
self.__fetch_comment(sym, database)
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(
self.comment.title, link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1 id="%s-page">%s</h1>' % (
id_from_text(self.title), self.title)
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def __redirect_if_needed(self, formatter, link_resolver):
redirect = self.meta.get("redirect")
if not redirect:
return False
link = link_resolver.get_named_link(redirect)
if link:
if formatter.extension.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
self_dir = os.path.dirname(page_path)
if self_dir == self.link.ref:
self_dir = ""
self.meta["redirect"] = os.path.relpath(link.ref, self_dir)
else:
warn('markdown-bad-link', "Bad redirect link '%s' in page: %s"
% (redirect, self.name))
return True
def __format_content(self, formatter, link_resolver):
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(
diag.code,
message=diag.message,
filename=self.source_file or self.name)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.name:
title = os.path.splitext(self.name)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(formatter.get_output_folder(self),
self.link.ref)
if not self.__redirect_if_needed(formatter, link_resolver):
self.__format_content(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.cache_page(self)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug('Formatting symbol %s in page %s' % (
symbol.unique_name, self.name), 'formatting')
symbol.detailed_description = formatter.format_symbol(
symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
database):
if sym:
self.__fetch_comment(sym, database)
all_syms.add(sym)
def __resolve_symbol(self, symbol, link_resolver, page_path):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (page_path, link.id_)
tsl = self.typed_symbols.get(type(symbol))
if tsl:
tsl.symbols.append(symbol)
by_parent_symbols = self.by_parent_symbols.get(symbol.parent_name)
if not by_parent_symbols:
by_parent_symbols = self.__get_empty_typed_symbols()
parent_name = symbol.parent_name
if parent_name is None:
parent_name = 'Others symbols'
self.by_parent_symbols[symbol.parent_name] = by_parent_symbols
by_parent_symbols.get(type(symbol)).symbols.append(symbol)
self.symbols.append(symbol)
debug('Resolved symbol %s to page %s' %
(symbol.unique_name, self.link.ref), 'resolution')
class GstExtension(Extension):
extension_name = 'gst-extension'
argument_prefix = 'gst'
__dual_links = {} # Maps myelement:XXX to GstMyElement:XXX
__parsed_cfiles = set()
__caches = {} # cachefile -> dict
__all_plugins_symbols = set()
def __init__(self, app, project):
super().__init__(app, project)
self.cache = {}
self.c_sources = []
self.cache_file = None
self.plugin = None
self.__elements = {}
self.__raw_comment_parser = GtkDocParser(project,
section_file_matching=False)
self.__plugins = None
self.__toplevel_comments = OrderedSet()
self.list_plugins_page = None
# If we have a plugin with only one element, we render it on the plugin
# page.
self.unique_feature = None
self.__on_index_symbols = []
# Links GTypeName to pagename for other-types so we now where to locate
# the symbols on creation.
self.__other_types_pages = {}
def _make_formatter(self):
return GstFormatter(self)
def create_symbol(self, *args, **kwargs):
sym = super().create_symbol(*args, **kwargs)
if self.unique_feature and sym:
self.__on_index_symbols.append(sym)
return sym
# pylint: disable=too-many-branches
def setup(self):
gi_extension = self.project.extensions.get('gi-extension')
self.gi_languages = [
lang.language_name for lang in gi_extension.languages
]
if not self.cache_file:
if self.list_plugins_page:
self.__plugins = self.create_symbol(
GstPluginsSymbol,
display_name="All " +
self.project.project_name.replace("-", " ").title(),
unique_name=self.project.project_name + "-all-gst-plugins",
plugins=[],
all_plugins=True)
super().setup()
return
gather_links()
comment_parser = GtkDocParser(self.project, False)
to_parse_sources = set(self.c_sources) - GstExtension.__parsed_cfiles
CCommentExtractor(self,
comment_parser).parse_comments(to_parse_sources)
GstExtension.__parsed_cfiles.update(self.c_sources)
self.debug("Parsing plugin %s, (cache file %s)" %
(self.plugin, self.cache_file))
if not self.cache:
error('setup-issue',
"No cache loaded or created for %s" % self.plugin)
plugins = []
if self.plugin:
pname = self.plugin
dot_idx = pname.rfind('.')
if dot_idx > 0:
pname = self.plugin[:dot_idx]
if pname.startswith('libgst'):
pname = pname[6:]
elif pname.startswith('gst'):
pname = pname[3:]
try:
plugin_node = {pname: self.cache[pname]}
except KeyError:
error('setup-issue', "Plugin %s not found" % pname)
else:
plugin_node = self.cache
for libfile, plugin in plugin_node.items():
plugin_sym = self.__parse_plugin(libfile, plugin)
if not plugin_sym:
continue
plugins.append(plugin_sym)
if not self.plugin:
self.__plugins = self.create_symbol(
GstPluginsSymbol,
display_name=self.project.project_name.replace("-",
" ").title(),
unique_name=self.project.project_name + "-gst-plugins",
plugins=plugins)
super().setup()
def _get_comment_smart_key(self, comment):
try:
return comment.title.description
except AttributeError:
return None
def _get_toplevel_comments(self):
if self.unique_feature:
return OrderedSet()
return self.__toplevel_comments
def get_plugin_comment(self):
if self.plugin:
res = self.app.database.get_comment("plugin-" + self.plugin)
return res
return None
def make_pages(self):
smart_pages = super().make_pages()
if not self.__plugins:
return None
if self.list_plugins_page is None:
index = smart_pages.get('gst-index')
if index is None:
return smart_pages
index.render_subpages = False
index.symbol_names.add(self.__plugins.unique_name)
for sym in self.__on_index_symbols:
index.symbol_names.add(sym.unique_name)
if self.unique_feature:
index.comment = self.app.database.get_comment(
"element-" + self.unique_feature)
else:
index.comment = self.get_plugin_comment()
return smart_pages
page = smart_pages.get(self.list_plugins_page)
page.render_subpages = False
page.extension_name = self.extension_name
page.symbol_names.add(self.__plugins.unique_name)
self.__plugins.plugins = self.__all_plugins_symbols
return smart_pages
def add_comment(self, comment):
# We handle toplevel comments ourself, make sure all comments
# end up in the database
comment.toplevel = False
super().add_comment(comment)
def _get_smart_index_title(self):
if self.plugin:
return self.__plugins.display_name
return 'GStreamer plugins documentation'
@staticmethod
def add_arguments(parser):
group = parser.add_argument_group('Gst extension', DESCRIPTION)
GstExtension.add_index_argument(group)
# GstExtension.add_order_generated_subpages(group)
GstExtension.add_sources_argument(group, prefix='gst-c')
group.add_argument('--gst-cache-file', default=None)
group.add_argument('--gst-list-plugins-page', default=None)
group.add_argument('--gst-plugin-name', default=None)
group.add_argument('--gst-plugins-path', default=None)
def parse_config(self, config):
self.c_sources = config.get_sources('gst_c')
self.cache_file = config.get('gst_cache_file')
self.plugin = config.get('gst_plugin_name')
self.list_plugins_page = config.get('gst_list_plugins_page', None)
info('Parsing config!')
self.cache = {}
if self.cache_file:
self.cache = GstExtension.__caches.get(self.cache_file)
if not self.cache:
try:
with open(self.cache_file) as _:
self.cache = json.load(_)
except FileNotFoundError:
pass
if self.cache is None:
self.cache = {}
GstExtension.__caches[self.cache_file] = self.cache
super().parse_config(config)
def _get_smart_key(self, symbol):
if self.unique_feature:
return None
if isinstance(symbol, GstPluginSymbol):
# PluginSymbol are rendered on the index page
return None
res = symbol.extra.get('gst-element-name')
if res:
res = res.replace("element-", "")
return res
def __create_symbol(self, gtype, symbol, pagename):
if symbol["kind"] in ["enum", "flags"]:
if pagename in self.__other_types_pages:
# The enum was defined by another type (not an element)
# this page will be rendered as any GObject so we need
# to specify its parent name
parent_name = pagename
else:
parent_name = None
return self.__create_enum_symbol(gtype,
symbol.get('values'),
pagename,
parent_name=parent_name)
if symbol["kind"] == "object":
return self.__create_classed_type(gtype, symbol)
if symbol["kind"] == "interface":
return self.__create_classed_type(gtype, symbol, True)
assert "Not reached" == "False"
return None
def _remember_symbol_type(self, gtype, pagename):
self.__other_types_pages[gtype] = pagename
return gtype
# pylint: disable=too-many-locals
# pylint: disable=too-many-arguments
def __create_signal_symbol(self,
obj,
parent_uniquename,
name,
signal,
pagename,
parent_name=None):
args = signal['args']
instance_type = obj['hierarchy'][0]
unique_name = "%s::%s" % (parent_uniquename, name)
aliases = ["%s::%s" % (instance_type, name)]
gi_extension = self.project.extensions.get('gi-extension')
python_lang = gi_extension.get_language('python')
args_type_names = [
(type_tokens_from_type_name('GstElement', python_lang), 'param_0')
]
for arg in args:
arg_name = arg["name"]
arg_type_name = self._remember_symbol_type(arg["type"], pagename)
type_tokens = type_tokens_from_type_name(arg_type_name,
python_lang)
args_type_names.append((type_tokens, arg_name))
if not signal.get('action'):
args_type_names.append(
(type_tokens_from_type_name('gpointer', python_lang), "udata"))
params = []
for comment_name in [unique_name] + aliases:
comment = self.app.database.get_comment(comment_name)
if comment:
for i, argname in enumerate(comment.params.keys()):
args_type_names[i] = (args_type_names[i][0], argname)
for tokens, argname in args_type_names:
params.append(ParameterSymbol(argname=argname, type_tokens=tokens))
return_type_name = self._remember_symbol_type(signal["return-type"],
pagename)
if return_type_name == 'void':
retval = [None]
else:
tokens = type_tokens_from_type_name(return_type_name, python_lang)
retval = [ReturnItemSymbol(type_tokens=tokens)]
if signal.get('action'):
typ = ActionSignalSymbol
for param in params:
self.add_attrs(param, action=True)
else:
typ = SignalSymbol
res = self.create_symbol(typ,
parameters=params,
return_value=retval,
display_name=name,
unique_name=unique_name,
extra={'gst-element-name': pagename},
aliases=aliases,
parent_name=parent_name)
if res:
flags = []
when = signal.get('when')
if when == "first":
flags.append(gi.flags.RunFirstFlag())
elif when == "last":
flags.append(gi.flags.RunLastFlag())
elif when == "cleanup":
flags.append(gi.flags.RunCleanupFlag())
no_hooks = signal.get('no-hooks')
if no_hooks:
flags.append(gi.flags.NoHooksFlag())
action = signal.get('action')
if action:
flags.append(gi.flags.ActionFlag())
# This is incorrect, it's not yet format time
extra_content = self.formatter.format_flags(flags)
res.extension_contents['Flags'] = extra_content
return res
def __create_signal_symbols(self,
obj,
parent_uniquename,
element_name,
parent_name=None):
res = []
signals = obj.get('signals', {})
if not signals:
return res
for name, signal in signals.items():
res.append(
self.__create_signal_symbol(obj,
parent_uniquename,
name,
signal,
element_name,
parent_name=parent_name))
return res
def __create_property_symbols(self,
obj,
parent_uniquename,
pagename,
parent_name=None):
res = []
properties = obj.get('properties', [])
if not properties:
return res
gi_extension = self.project.extensions.get('gi-extension')
python_lang = gi_extension.get_language('python')
for name, prop in properties.items():
unique_name = '%s:%s' % (obj.get('name', parent_uniquename), name)
flags = [ReadableFlag()]
if prop['writable']:
flags += [WritableFlag()]
if prop['construct-only']:
flags += [ConstructOnlyFlag()]
elif prop['construct']:
flags += [ConstructFlag()]
prop_type_name = self._remember_symbol_type(prop["type"], pagename)
tokens = type_tokens_from_type_name(prop_type_name, python_lang)
type_ = QualifiedSymbol(type_tokens=tokens)
default = prop.get('default')
if obj['hierarchy'][0] != parent_uniquename:
aliases = ['%s:%s' % (obj['hierarchy'][0], name)]
else:
aliases = []
sym = self.app.database.get_symbol(unique_name)
if sym is None:
sym = self.create_symbol(
PropertySymbol,
prop_type=type_,
display_name=name,
unique_name=unique_name,
aliases=aliases,
parent_name=parent_name,
extra={'gst-element-name': pagename},
)
assert sym
if not self.app.database.get_comment(unique_name):
comment = Comment(unique_name,
Comment(name=name),
description=prop['blurb'])
self.app.database.add_comment(comment)
# FIXME This is incorrect, it's not yet format time (from gi_extension)
extra_content = self.formatter.format_flags(flags)
sym.extension_contents['Flags'] = extra_content
if default:
if prop_type_name in ['GstCaps', 'GstStructure']:
default = '<pre class="language-yaml">' + \
'<code class="language-yaml">%s</code></pre>' % default
sym.extension_contents['Default value'] = default
res.append(sym)
return res
def __create_enum_symbol(self,
type_name,
enum,
pagename,
parent_name=None):
display_name = re.sub(r"([a-z])([A-Z])", r"\g<1>-\g<2>",
type_name.replace('Gst', ''))
unique_name = type_name
if self.app.database.get_symbol(unique_name):
# Still required as some bin manually proxy children properties inside
# themselves (like GstFakeSinkStateError in fakevideosink for example)
unique_name = pagename + '_' + type_name
members = []
for val in enum:
value_unique_name = "%s::%s" % (type_name, val['name'])
if self.app.database.get_symbol(value_unique_name):
value_unique_name = "%s_%s" % (pagename, value_unique_name)
member_sym = self.create_symbol(
GstNamedConstantValue,
unique_name=value_unique_name,
display_name=val['name'],
value=val['value'],
parent_name=parent_name,
val=val,
extra={'gst-element-name': pagename})
if member_sym:
members.append(member_sym)
symbol = self.create_symbol(GstNamedConstantsSymbols,
anonymous=False,
raw_text=None,
display_name=display_name.capitalize(),
unique_name=unique_name,
parent_name=parent_name,
members=members,
extra={'gst-element-name': pagename})
if not symbol:
return None
symbol.values = enum
return symbol
def __create_hierarchy(self, pagename, element_dict):
hierarchy = []
for klass_name in element_dict["hierarchy"][1:]:
self._remember_symbol_type(klass_name, pagename)
link = Link(None, klass_name, klass_name, mandatory=True)
sym = QualifiedSymbol(type_tokens=[link])
hierarchy.append(sym)
hierarchy.reverse()
return hierarchy
def __create_classed_type(self, pagename, _object, is_interface=False):
unique_name = _object['hierarchy'][0]
properties = self.__create_property_symbols(_object,
unique_name,
pagename,
parent_name=unique_name)
signals = self.__create_signal_symbols(_object,
unique_name,
pagename,
parent_name=unique_name)
return self.create_symbol(
GIInterfaceSymbol if is_interface else GIClassSymbol,
hierarchy=self.__create_hierarchy(pagename, _object),
display_name=unique_name,
unique_name=unique_name,
parent_name=unique_name,
properties=properties,
signals=signals,
extra={'gst-element-name': pagename})
def __create_pad_template_symbols(self, element, plugin_name):
templates = element.get('pad-templates', {})
res = []
if not templates:
return res
for tname, template in templates.items():
name = tname.replace("%%", "%")
unique_name = '%s!%s' % (element['hierarchy'][0], name)
pagename = 'element-' + element['name']
gtype = self._remember_symbol_type(template.get("type", "GstPad"),
pagename)
link = Link(None, gtype, gtype, mandatory=True)
object_type = QualifiedSymbol(type_tokens=[link])
res.append(
self.create_symbol(GstPadTemplateSymbol,
name=name,
direction=template["direction"],
presence=template["presence"],
caps=template["caps"],
filename=plugin_name,
parent_name=None,
object_type=object_type,
display_name=name,
unique_name=unique_name,
extra={'gst-element-name': pagename}))
return res
def __extract_feature_comment(self, feature_type, feature):
pagename = feature_type + '-' + feature['name']
possible_comment_names = [pagename, feature['name']]
if feature_type == 'element':
possible_comment_names.append(feature['hierarchy'][0])
comment = None
for comment_name in possible_comment_names:
comment = self.app.database.get_comment(comment_name)
if comment:
break
description = feature.get('description')
if not comment:
comment = Comment(
pagename,
Comment(description=feature['name']),
description=description,
short_description=Comment(description=description))
self.app.database.add_comment(comment)
elif not comment.short_description:
comment.short_description = Comment(description=description)
comment.title = Comment(description=feature['name'])
comment.name = feature.get('name', pagename)
comment.meta['title'] = feature['name']
self.__toplevel_comments.add(comment)
return pagename, comment
def __parse_plugin(self, plugin_name, plugin):
elements = []
feature_names = list(plugin.get('elements', {}).keys()) \
+ list(plugin.get('tracers', {}).keys()) \
+ list(plugin.get('device-providers', {}).keys())
self.__other_types_pages = {}
if self.plugin and len(feature_names) == 1:
self.unique_feature = feature_names[0]
for ename, tracer in plugin.get('tracers', {}).items():
tracer['name'] = ename
self.__extract_feature_comment("tracer", tracer)
other_types = []
for provider_name, provider in plugin.get('device-providers',
{}).items():
provider['name'] = provider_name
_, comment = self.__extract_feature_comment("provider", provider)
comment.description += """\n\n# Provided device example"""
other_types.append(
self.__create_classed_type(provider_name,
provider.get('device-example')))
for ename, element in plugin.get('elements', {}).items():
element['name'] = ename
pagename, _ = self.__extract_feature_comment("element", element)
interfaces = []
for interface in element.get("interfaces", []):
self._remember_symbol_type(interface, pagename)
interfaces.append(
QualifiedSymbol(type_tokens=[
Link(None, interface, interface, mandatory=True)
]))
aliases = [pagename, element['hierarchy'][0]]
sym = self.create_symbol(GstElementSymbol,
parent_name=None,
display_name=element['name'],
hierarchy=self.__create_hierarchy(
pagename, element),
unique_name=element['name'],
filename=plugin_name,
extra={'gst-element-name': pagename},
rank=str(element['rank']),
author=element['author'],
classification=element['klass'],
plugin=plugin_name,
aliases=aliases,
package=plugin['package'],
interfaces=interfaces)
if not sym:
continue
self.__elements[element['name']] = sym
sym.properties.extend(
self.__create_property_symbols(element, element['name'],
pagename))
sym.signals.extend(
self.__create_signal_symbols(element, element['name'],
pagename))
sym.pad_templates.extend(
self.__create_pad_template_symbols(element, plugin_name))
elements.append(sym)
types = list(plugin['other-types'].items())
while True:
type_ = None
for tmptype in types:
if tmptype[0] in self.__other_types_pages:
type_ = tmptype
break
if not type_:
break
types.remove(type_)
other_types.append(
self.__create_symbol(type_[0], type_[1],
self.__other_types_pages[type_[0]]))
for _type in types:
self.warn(
"no-location-indication",
"Type %s has been marked with `gst_type_mark_as_plugin_api`"
" but is not used in any of %s API (it might require to"
" be manually removed from the cache in case of plugin"
" move)" % (_type[0], plugin_name))
plugin = self.create_symbol(
GstPluginSymbol,
description=plugin['description'],
display_name=plugin_name,
unique_name='plugin-' + plugin_name,
license=plugin['license'],
package=plugin['package'],
filename=plugin['filename'],
elements=elements,
other_types=other_types,
extra={'gst-plugins': 'plugins-' + plugin['filename']})
if not plugin:
return None
self.__all_plugins_symbols.add(plugin)
if self.plugin:
self.__plugins = plugin
return plugin
def __get_link_cb(self, resolver, name):
link = self.__dual_links.get(name)
if link:
# Upsert link on the first run
if isinstance(link, str):
sym = self.app.database.get_symbol(link)
link = sym.link
if link:
return link
return (resolver, name)
def format_page(self, page, link_resolver, output):
link_resolver.get_link_signal.connect(search_online_links)
gi_extension = self.project.extensions.get('gi-extension')
page.meta['extra']['gi-languages'] = self.gi_languages
super().format_page(page, link_resolver, output)
link_resolver.get_link_signal.disconnect(search_online_links)
class TestTree(unittest.TestCase):
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(
os.path.join(here, 'tmp-markdown-files'))
self.private_folder = os.path.abspath(os.path.join(
here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.private_folder)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.database = Database()
self.database.setup(self.private_folder)
self.link_resolver = LinkResolver(self.database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.project_name = 'test-project'
self.sanitized_name = 'test-project-0.1'
self.incremental = False
self.tree = Tree(self, self)
self.test_ext = TestExtension(self, self)
self.core_ext = CoreExtension(self, self)
cfg = Config()
self.test_ext.parse_config(cfg)
self.core_ext.parse_config(cfg)
self.subprojects = {}
self.is_toplevel = True
def tearDown(self):
self.__remove_tmp_dirs()
del self.test_ext
del self.core_ext
def get_generated_doc_folder(self):
return os.path.join(self.private_folder, 'generated')
def get_base_doc_folder(self):
return self.__md_dir
def get_private_folder(self):
return self.private_folder
def __parse_sitemap(self, text):
path = os.path.join(self.__md_dir, 'sitemap.txt')
with io.open(path, 'w', encoding='utf-8') as _:
_.write(text)
return self.sitemap_parser.parse(path)
def __create_md_file(self, name, contents):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
_.write(contents)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
def __create_src_file(self, name, symbols):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
for symbol in symbols:
_.write('%s\n' % symbol)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
return path
def __remove_src_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __remove_md_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __touch_src_file(self, name):
path = os.path.join(self.__md_dir, name)
touch(path)
def __remove_tmp_dirs(self):
shutil.rmtree(self.__md_dir, ignore_errors=True)
shutil.rmtree(self.private_folder, ignore_errors=True)
shutil.rmtree(self.__src_dir, ignore_errors=True)
shutil.rmtree(self.__output_dir, ignore_errors=True)
shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)
def test_basic(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'))
self.tree.parse_sitemap(sitemap)
pages = self.tree.get_pages()
# We do not care about ordering
self.assertSetEqual(set(pages.keys()),
set([u'index.markdown', u'section.markdown']))
index = pages.get('index.markdown')
self.assertEqual(index.title, u'My documentation')
def test_basic_incremental(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'))
self.tree.parse_sitemap(sitemap)
# Building from scratch, all pages are stale
self.assertSetEqual(set(self.tree.get_stale_pages()),
set([u'index.markdown', u'section.markdown']))
self.tree.persist()
self.incremental = True
self.tree = Tree(self, self)
self.tree.parse_sitemap(sitemap)
# Nothing changed, no page is stale
self.assertSetEqual(set(self.tree.get_stale_pages()), set({}))
# But we still have our pages
self.assertSetEqual(set(self.tree.get_pages()),
set([u'index.markdown', u'section.markdown']))
touch(os.path.join(self.__md_dir, u'section.markdown'))
self.tree = Tree(self, self)
self.tree.parse_sitemap(sitemap)
self.assertSetEqual(set(self.tree.get_stale_pages()),
set([u'section.markdown']))
def __assert_extension_names(self, tree, name_map):
pages = tree.get_pages()
for name, ext_name in list(name_map.items()):
page = pages[name]
self.assertEqual(ext_name, page.extension_name)
def __assert_stale(self, expected_stale):
stale_pages = self.tree.get_stale_pages()
for pagename in expected_stale:
self.assertIn(pagename, stale_pages)
stale_pages.pop(pagename)
self.assertEqual(len(stale_pages), 0)
def __create_test_layout(self, with_ext_index=True, sitemap=None):
if not sitemap:
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
else:
inp = sitemap
sources = []
sources.append(
self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2']))
sources.append(
self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4']))
if with_ext_index:
self.test_ext.index = 'test-index.markdown'
self.test_ext.sources = sources
self.test_ext.setup()
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('core_page.markdown',
(u'# My non-extension page\n'))
if with_ext_index:
self.__create_md_file('test-index.markdown',
(u'# My test index\n'))
self.__create_md_file(
'test-section.markdown',
(u'# My test section\n'
'\n'
'Linking to [a generated page](source_a.test)\n'))
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__create_md_file('page_y.markdown', (u'# Page Y\n'))
self.tree.parse_sitemap(sitemap)
return sitemap
def __update_test_layout(self, sitemap):
self.test_ext.reset()
self.tree = Tree(self, self)
self.test_ext.setup()
self.tree.parse_sitemap(sitemap)
def test_extension_basic(self):
_ = self.__create_test_layout()
self.__assert_extension_names(
self.tree, {
u'index.markdown': 'core',
u'test-index': 'test-extension',
u'test-section.markdown': 'test-extension',
u'source_a.test': 'test-extension',
u'source_b.test': 'test-extension',
u'page_x.markdown': 'test-extension',
u'page_y.markdown': 'test-extension',
u'core_page.markdown': 'core'
})
all_pages = self.tree.get_pages()
self.assertEqual(len(all_pages), 8)
self.__assert_stale(all_pages)
self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
self.assertIn('source_a.test',
all_pages['test-section.markdown'].subpages)
def test_extension_override(self):
self.__create_md_file('source_a.test.markdown', (u'# My override\n'))
_ = self.__create_test_layout()
page = self.tree.get_pages()['source_a.test']
self.assertEqual(page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
self.assertEqual(os.path.basename(page.source_file),
'source_a.test.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(out, u'<h1>My override</h1>\n')
def test_no_extension_index_override(self):
_ = self.__create_test_layout(with_ext_index=False)
ext_index = self.tree.get_pages()['test-index']
self.assertEqual(ext_index.generated, True)
self.assertEqual(len(ext_index.subpages), 4)
def test_parse_yaml(self):
inp = (u'index.markdown\n')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown',
(u'---\n'
'title: A random title\n'
'symbols: [symbol_1, symbol_2]\n'
'...\n'
'# My documentation\n'))
self.tree.parse_sitemap(sitemap)
pages = self.tree.get_pages()
page = pages.get('index.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(out, u'<h1>My documentation</h1>\n')
self.assertEqual(page.title, u'A random title')
self.assertEqual(page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
def test_empty_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'
'\n'
'[](index.markdown)\n'))
self.tree.parse_sitemap(sitemap)
self.tree.resolve_symbols(self.database, self.link_resolver)
self.tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = self.tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html">My documentation</a></p>\n')
def test_labeled_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown',
(u'# My section\n'
'\n'
'[a label](index.markdown)\n'))
self.tree.parse_sitemap(sitemap)
self.tree.resolve_symbols(self.database, self.link_resolver)
self.tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = self.tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html">a label</a></p>\n')
def test_anchored_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown#subsection)\n'))
self.tree.parse_sitemap(sitemap)
self.tree.resolve_symbols(self.database, self.link_resolver)
self.tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = self.tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html#subsection">My documentation</a></p>\n')
# pylint: disable=too-many-statements
def test_extension_incremental(self):
sitemap = self.__create_test_layout()
self.tree.persist()
self.incremental = True
# Here we touch source_a.test, as its symbols were
# all contained in a generated page, only that page
# should now be stale
self.__touch_src_file('source_a.test')
self.__update_test_layout(sitemap)
self.__assert_stale(set(['source_a.test']))
self.tree.persist()
# We now touch source_b.test, which symbols are contained
# both in a generated page and a user-provided one.
# We expect both pages to be stale
self.__touch_src_file('source_b.test')
self.__update_test_layout(sitemap)
self.__assert_stale(set(['source_b.test', 'page_x.markdown']))
self.tree.persist()
# This one is trickier: we unlist symbol_3 from
# page_x, which means the symbol should now be
# documented in the generated page for source_b.test.
# We expect both pages to be stale, and make sure
# they contain the right symbols
self.__create_md_file('page_x.markdown', (u'# Page X\n'))
self.__update_test_layout(sitemap)
self.__assert_stale(set(['source_b.test', 'page_x.markdown']))
page_x = self.tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet())
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
self.tree.persist()
# Let's make sure the opposite use case works as well,
# we relocate symbol_3 in page_x , both page_x and
# the generated page for source_b.test should be stale
# and the symbols should be back to their original
# layout.
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__update_test_layout(sitemap)
self.__assert_stale(set(['source_b.test', 'page_x.markdown']))
page_x = self.tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
self.tree.persist()
# We now move the definition of symbol_3 to source_a.test,
# we thus expect the generated page for source_a.test to be
# stale because its source changed, same for source_b.test,
# and page_x.markdown should be stale as well because the
# definition of symbol_3 may have changed. The final
# symbol layout should not have changed however.
self.__create_src_file('source_a.test',
['symbol_1', 'symbol_2', 'symbol_3'])
self.__create_src_file('source_b.test', ['symbol_4'])
self.__update_test_layout(sitemap)
self.__assert_stale(
set(['source_a.test', 'source_b.test', 'page_x.markdown']))
page_x = self.tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
source_a_page = self.tree.get_pages()['source_a.test']
self.assertEqual(source_a_page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
self.tree.persist()
# And we rollback again
self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2'])
self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4'])
self.__update_test_layout(sitemap)
self.__assert_stale(
set(['source_a.test', 'source_b.test', 'page_x.markdown']))
page_x = self.tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
source_a_page = self.tree.get_pages()['source_a.test']
self.assertEqual(source_a_page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
self.tree.persist()
# Now we'll try removing page_x altogether
self.__remove_md_file('page_x.markdown')
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
new_sitemap = self.__parse_sitemap(inp)
self.__update_test_layout(new_sitemap)
self.__assert_stale(set(['source_b.test']))
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
self.tree.persist()
# And rollback again
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__update_test_layout(sitemap)
self.__assert_stale(set(['page_x.markdown', 'source_b.test']))
page_x = self.tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = self.tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
self.tree.persist()
def test_index_override_incremental(self):
sitemap = self.__create_test_layout()
self.tree.persist()
index_page = self.tree.get_pages()['test-index']
self.assertIn('source_b.test', index_page.subpages)
self.incremental = True
self.__touch_src_file('test-index.markdown')
self.__update_test_layout(sitemap)
index_page = self.tree.get_pages()['test-index']
self.assertIn('source_b.test', index_page.subpages)
self.tree.persist()
def test_extension_index_only(self):
inp = (u'test-index\n' '\ttest-section.markdown\n')
self.test_ext.setup()
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('test-section.markdown', u'# My test section\n')
self.tree.parse_sitemap(sitemap)
self.__assert_extension_names(
self.tree, {
u'test-index': 'test-extension',
u'test-section.markdown': 'test-extension'
})
def test_extension_auto_sorted_override(self):
self.__create_md_file('source_b.test.markdown',
(u'---\nauto-sort: true\n...\n# My override\n'))
sitemap = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_b.test\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
_ = self.__create_test_layout(sitemap=sitemap)
pages = self.tree.get_pages()
self.assertTrue(pages['source_a.test'].pre_sorted)
self.assertFalse(pages['source_b.test'].pre_sorted)
self.__create_md_file('source_b.test.markdown', (u'# My override\n'))
sitemap = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_b.test\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
_ = self.__create_test_layout(sitemap=sitemap)
pages = self.tree.get_pages()
self.assertTrue(pages['source_a.test'].pre_sorted)
self.assertTrue(pages['source_b.test'].pre_sorted)
def test_extension_implicit_override(self):
self.__create_md_file(
'source_b.test.markdown',
(u'---\nsymbols:\n - symbol_2\n...\n# My override\n'))
_ = self.__create_test_layout()
source_b = self.tree.get_pages()['source_b.test']
self.assertEqual(os.path.basename(source_b.source_file),
'source_b.test.markdown')
self.assertEqual(source_b.symbol_names, ['symbol_2', 'symbol_4'])
source_a = self.tree.get_pages()['source_a.test']
self.assertEqual(source_a.symbol_names, ['symbol_1'])
out, _ = cmark.ast_to_html(source_b.ast, None)
self.assertEqual(out, u'<h1>My override</h1>\n')
class Page(object):
"Banana banana"
meta_schema = {
Optional('title'): And(str, len),
Optional('symbols'): Schema([And(str, len)]),
Optional('short-description'): And(str, len),
Optional('render-subpages'): bool,
Optional('auto-sort'): bool,
Optional('full-width'): bool
}
# pylint: disable=too-many-arguments
def __init__(self,
source_file,
ast,
output_path,
project_name,
meta=None,
raw_contents=None):
"Banana banana"
assert source_file
basename = os.path.basename(source_file)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.pre_sorted = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.project_name = project_name
self.cached_paths = OrderedSet()
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.render_subpages = meta.get('render-subpages', True)
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, ref)
def __getstate__(self):
return {
'ast': None,
'build_path': None,
'title': self.title,
'raw_contents': self.raw_contents,
'short_description': self.short_description,
'extension_name': self.extension_name,
'link': self.link,
'meta': self.meta,
'source_file': self.source_file,
'comment': self.comment,
'generated': self.generated,
'is_stale': False,
'formatted_contents': None,
'detailed_description': None,
'output_attrs': None,
'symbols': [],
'typed_symbols': {},
'subpages': self.subpages,
'symbol_names': self.symbol_names,
'project_name': self.project_name,
'pre_sorted': self.pre_sorted,
'cached_paths': self.cached_paths,
'render_subpages': self.render_subpages
}
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
typed_symbols_list = namedtuple('TypedSymbolsList',
['name', 'symbols'])
for subclass in all_subclasses(Symbol):
self.typed_symbols[subclass] = typed_symbols_list(
subclass.get_plural_name(), [])
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
for sym_type in [
ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.source_file)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
# pylint: disable=no-self-use
def __fetch_comment(self, sym, database):
old_comment = sym.comment
new_comment = database.get_comment(sym.unique_name)
sym.comment = Comment(sym.unique_name)
if new_comment:
sym.comment = new_comment
elif old_comment:
if old_comment.filename not in (ChangeTracker.all_stale_files |
ChangeTracker.all_unlisted_files):
sym.comment = old_comment
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(self.comment.title,
link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1>%s</h1>' % self.title
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(formatter.get_output_folder(self),
self.link.ref)
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.cache_page(self)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug(
'Formatting symbol %s in page %s' %
(symbol.unique_name, self.source_file), 'formatting')
symbol.skip = not formatter.format_symbol(symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
database):
if sym:
self.__fetch_comment(sym, database)
new_symbols = sum(tree.resolving_symbol_signal(self, sym), [])
all_syms.add(sym)
for symbol in new_symbols:
self.__query_extra_symbols(symbol, all_syms, tree,
link_resolver, database)
def __resolve_symbol(self, symbol, link_resolver, page_path):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (page_path, link.id_)
tsl = self.typed_symbols.get(type(symbol))
if tsl:
tsl.symbols.append(symbol)
self.symbols.append(symbol)
debug(
'Resolved symbol %s to page %s' %
(symbol.display_name, self.link.ref), 'resolution')
class Page(object):
"Banana banana"
meta_schema = {Optional('title'): And(unicode, len),
Optional('symbols'): Schema([And(unicode, len)]),
Optional('short-description'): And(unicode, len)}
resolving_symbol_signal = Signal()
formatting_signal = Signal()
def __init__(self, source_file, ast, meta=None, raw_contents=None):
"Banana banana"
assert source_file
if os.path.isabs(source_file):
basename = os.path.basename(source_file)
else:
basename = source_file.replace('/', '-')
name = os.path.splitext(basename)[0]
pagename = '%s.html' % name
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file,
str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, name)
def __getstate__(self):
return {'ast': None,
'title': self.title,
'raw_contents': self.raw_contents,
'short_description': self.short_description,
'extension_name': self.extension_name,
'link': self.link,
'meta': self.meta,
'source_file': self.source_file,
'comment': self.comment,
'generated': self.generated,
'is_stale': False,
'formatted_contents': None,
'detailed_description': None,
'output_attrs': None,
'symbols': [],
'typed_symbols': {},
'subpages': self.subpages,
'symbol_names': self.symbol_names}
def resolve_symbols(self, doc_database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `doc_database`, and added to lists of actual symbols, sorted
by symbol class.
"""
typed_symbols_list = namedtuple(
'TypedSymbolsList', ['name', 'symbols'])
self.typed_symbols[Symbol] = typed_symbols_list('FIXME symbols', [])
self.typed_symbols[FunctionSymbol] = typed_symbols_list(
"Functions", [])
self.typed_symbols[CallbackSymbol] = typed_symbols_list(
"Callback Functions", [])
self.typed_symbols[FunctionMacroSymbol] = typed_symbols_list(
"Function Macros", [])
self.typed_symbols[ConstantSymbol] = typed_symbols_list(
"Constants", [])
self.typed_symbols[ExportedVariableSymbol] = typed_symbols_list(
"Exported Variables", [])
self.typed_symbols[StructSymbol] = typed_symbols_list(
"Data Structures", [])
self.typed_symbols[EnumSymbol] = typed_symbols_list("Enumerations", [])
self.typed_symbols[AliasSymbol] = typed_symbols_list("Aliases", [])
self.typed_symbols[SignalSymbol] = typed_symbols_list("Signals", [])
self.typed_symbols[PropertySymbol] = typed_symbols_list(
"Properties", [])
self.typed_symbols[VFunctionSymbol] = typed_symbols_list(
"Virtual Methods", [])
self.typed_symbols[ClassSymbol] = typed_symbols_list("Classes", [])
self.typed_symbols[InterfaceSymbol] = typed_symbols_list("Interfaces", [])
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = doc_database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, link_resolver, doc_database)
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver)
self.symbol_names.add(sym.unique_name)
class_syms = self.typed_symbols[ClassSymbol].symbols
interface_syms = self.typed_symbols[InterfaceSymbol].symbols
struct_syms = self.typed_symbols[StructSymbol].symbols
if self.title is None:
if class_syms:
self.title = class_syms[0].display_name
elif interface_syms:
self.title = interface_syms[0].display_name
elif struct_syms:
self.title = struct_syms[0].display_name
if self.comment is None:
if class_syms and class_syms[0].comment:
self.comment = class_syms[0].comment
elif interface_syms and interface_syms[0].comment:
self.comment = interface_syms[0].comment
elif struct_syms and struct_syms[0].comment:
self.comment = struct_syms[0].comment
# pylint: disable=no-self-use
def __fetch_comment(self, sym, doc_database):
old_comment = sym.comment
new_comment = doc_database.get_comment(sym.unique_name)
sym.comment = Comment(sym.unique_name)
if new_comment:
sym.comment = new_comment
elif old_comment:
if not old_comment.filename in (ChangeTracker.all_stale_files |
ChangeTracker.all_unlisted_files):
sym.comment = old_comment
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(
self.comment.title, link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1>%s</h1>' % self.title
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(
diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
else:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
Page.formatting_signal(self, formatter)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.write_page(self, output)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug('Formatting symbol %s in page %s' % (
symbol.unique_name, self.source_file), 'formatting')
symbol.skip = not formatter.format_symbol(symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, link_resolver, doc_database):
if sym:
self.__fetch_comment(sym, doc_database)
new_symbols = sum(Page.resolving_symbol_signal(self, sym),
[])
all_syms.add(sym)
for symbol in new_symbols:
self.__query_extra_symbols(symbol, all_syms, link_resolver, doc_database)
def __resolve_symbol(self, symbol, link_resolver):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (self.link.ref, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (self.link.ref, link.id_)
tsl = self.typed_symbols[type(symbol)]
tsl.symbols.append(symbol)
self.symbols.append(symbol)
debug('Resolved symbol %s to page %s' %
(symbol.display_name, self.link.ref), 'resolution')