def parse_config(self, config, toplevel=False):
"""Parses @config setting up @self state."""
self.sitemap_path = config.get_path('sitemap')
if self.sitemap_path is None:
error('invalid-config', 'No sitemap was provided')
self.include_paths = OrderedSet([])
index_file = config.get_index()
if index_file:
if not os.path.exists(index_file):
error('invalid-config',
'The provided index "%s" does not exist' % index_file)
self.include_paths |= OrderedSet([os.path.dirname(index_file)])
self.include_paths |= OrderedSet(config.get_paths('include_paths'))
self.is_toplevel = toplevel
self.tree = Tree(self, self.app)
self.__create_extensions()
for extension in list(self.extensions.values()):
if toplevel:
extension.parse_toplevel_config(config)
extension.parse_config(config)
if not toplevel and config.conf_file:
self.app.change_tracker.add_hard_dependency(config.conf_file)
self.extra_asset_folders = OrderedSet(config.get_paths('extra_assets'))
def __update_meta(self, meta):
for key, value in meta.items():
try:
self.meta.update(
Schema(Page.meta_schema).validate(
{key.replace('_', '-').lower(): value}))
except SchemaError as _:
warn(
'invalid-page-metadata',
'%s: Invalid metadata: \n%s, discarding metadata' %
(self.source_file, str(_)))
if not self.meta.get('extra'):
self.meta['extra'] = defaultdict()
self.title = meta.get('title', self.title)
self.thumbnail = meta.get('thumbnail')
self.listed_symbols = OrderedSet(
meta.get('symbols') or self.symbol_names)
self.private_symbols = OrderedSet(
meta.get('private-symbols') or self.private_symbols)
self.symbol_names = OrderedSet(
meta.get('symbols') or self.symbol_names)
self.short_description = meta.get('short-description',
self.short_description)
self.render_subpages = meta.get('render-subpages',
self.render_subpages)
def prepare_page_attributes(self, page):
"""
Banana banana
"""
page.output_attrs['html']['scripts'] = OrderedSet()
page.output_attrs['html']['stylesheets'] = OrderedSet()
page.output_attrs['html']['extra_html'] = []
page.output_attrs['html']['extra_footer_html'] = []
if HtmlFormatter.add_anchors:
page.output_attrs['html']['scripts'].add(
os.path.join(HERE, 'html_assets', 'css.escape.js'))
Formatter.prepare_page_attributes(self, page)
def __resolve_patterns(self, source_patterns, from_conf):
if source_patterns is None:
return OrderedSet()
all_files = OrderedSet()
for item in source_patterns:
item = self.__abspath(item, from_conf)
if '*' in item:
all_files |= glob.glob(item)
else:
all_files.add(item)
return all_files
def prepare_page_attributes(self, page):
"""
Banana banana
"""
self._current_page = page
page.output_attrs['html']['scripts'] = OrderedSet()
page.output_attrs['html']['stylesheets'] = OrderedSet()
page.output_attrs['html']['extra_html'] = []
page.output_attrs['html']['edit_button'] = ''
page.output_attrs['html']['extra_footer_html'] = []
if Formatter.add_anchors:
page.output_attrs['html']['scripts'].add(
os.path.join(HERE, 'assets', 'css.escape.js'))
def __init__(self,
source_file,
ast,
output_path,
project_name,
meta=None,
raw_contents=None):
"Banana banana"
assert source_file
basename = os.path.basename(source_file)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.pre_sorted = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.project_name = project_name
self.cached_paths = OrderedSet()
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.render_subpages = meta.get('render-subpages', True)
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, ref)
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(
os.path.join(here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def get_stale_files(self, all_files, fileset_name):
"""
Banana banana
"""
stale = OrderedSet()
previous_mtimes = self.mtimes[fileset_name]
new_mtimes = defaultdict()
for filename in all_files:
mtime = get_mtime(filename)
prev_mtime = previous_mtimes.pop(filename, None)
new_mtimes[filename] = mtime
if mtime == prev_mtime:
continue
stale.add(filename)
self.mtimes[fileset_name] = new_mtimes
unlisted = set(previous_mtimes.keys())
ChangeTracker.all_stale_files |= stale
ChangeTracker.all_unlisted_files |= unlisted
return stale, unlisted
def get_pagename(self, name):
self.__find_package_root()
for path in OrderedSet([self.__package_root]) | self.source_roots:
commonprefix = os.path.commonprefix([path, name])
if commonprefix == path:
return os.path.relpath(name, path)
return name
def __init__(self, app, dependency_map=None, page_map=None):
self.app = app
self.tree = None
self.include_paths = None
self.extensions = OrderedDict()
self.tag_validators = {}
self.project_name = None
self.project_version = None
self.sanitized_name = None
self.sitemap_path = None
self.subprojects = {}
self.extra_asset_folders = OrderedSet()
self.extra_assets = {}
if dependency_map is None:
self.dependency_map = {}
else:
self.dependency_map = dependency_map
if page_map is None:
self.page_map = {}
else:
self.page_map = page_map
if os.name == 'nt':
self.datadir = os.path.join(os.path.dirname(__file__), '..',
'share')
else:
self.datadir = "/usr/share"
self.formatted_signal = Signal()
self.written_out_signal = Signal()
self.is_toplevel = False
def parse_config(self, config):
"""
Override this, making sure to chain up first, if your extension adds
its own custom command line arguments, or you want to do any further
processing on the automatically added arguments.
The default implementation will set attributes on the extension:
- 'sources': a set of absolute paths to source files for this extension
- 'index': absolute path to the index for this extension
Additionally, it will set an attribute for each argument added with
`Extension.add_path_argument` or `Extension.add_paths_argument`, with
the extension's `Extension.argument_prefix` stripped, and dashes
changed to underscores.
Args:
config: a `config.Config` instance
"""
prefix = self.argument_prefix
self.sources = config.get_sources(prefix)
self.smart_sources = [
self._get_smart_filename(s) for s in self.sources]
self.index = config.get_index(prefix)
self.source_roots = OrderedSet(
config.get_paths('%s_source_roots' % prefix))
for arg, dest in list(self.paths_arguments.items()):
val = config.get_paths(arg)
setattr(self, dest, val)
for arg, dest in list(self.path_arguments.items()):
val = config.get_path(arg)
setattr(self, dest, val)
self.formatter.parse_config(config)
def _resolve_placeholder(self, tree, name, include_paths):
self.__find_package_root()
if name == '%s-index' % self.argument_prefix:
if self.index:
path = find_file(self.index, include_paths)
if path is None:
self.error("invalid-config",
"Could not find index file %s" % self.index)
return PageResolutionResult(True, path, None,
self.extension_name)
return PageResolutionResult(True, None, None, self.extension_name)
if self.smart_index:
for path in OrderedSet([self.__package_root]) | self.source_roots:
possible_path = os.path.join(path, name)
if possible_path in self._get_all_sources():
override_path = find_file('%s.markdown' % name,
include_paths)
if override_path:
return PageResolutionResult(True, override_path, None,
None)
return PageResolutionResult(
True, None, self.__get_rel_source_path(possible_path),
None)
return None
def test_smart_key_default(self):
sources = []
sources.append(self._create_src_file('source_a.test', ['symbol_1']))
self.test_ext.sources = sources
self.test_ext.setup()
self.assertDictEqual(self.test_ext.get_created_symbols(),
{sources[0]: OrderedSet(['symbol_1'])})
def get_dependencies(self):
"""
Retrieve the set of all dependencies for a given configuration.
Returns:
utils.utils.OrderedSet: The set of all dependencies for the
tracked configuration.
"""
all_deps = OrderedSet()
for key, _ in list(self.__config.items()):
if key in self.__cli:
continue
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1 - 1])
for key, _ in list(self.__cli.items()):
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1 - 1])
if self.conf_file is not None:
all_deps.add(self.conf_file)
all_deps.add(self.get_path("sitemap", rel_to_cwd=True))
cwd = os.getcwd()
return [os.path.relpath(fname, cwd) for fname in all_deps if fname]
def test_parse_yaml(self):
inp = (u'index.markdown\n')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'---\n'
'title: A random title\n'
'symbols: [symbol_1, symbol_2]\n'
'...\n'
'# My documentation\n'))
self.tree.parse_sitemap(sitemap)
pages = self.tree.get_pages()
page = pages.get('index.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(
out,
u'<h1>My documentation</h1>\n')
self.assertEqual(page.title, u'A random title')
self.assertEqual(
page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
def __find_structure_pagename(self, node, unique_name, is_class):
filename = self.__get_symbol_filename(unique_name, node)
if filename != self.__default_page:
return filename
if self.__symbol_is_relocated(unique_name, None):
return self.__default_page
if not is_class:
sym = self.__class_gtype_structs.get(node.attrib['name'])
if sym and sym.filename:
return sym.filename
filenames = []
for cnode in node:
cunique_name = get_symbol_names(cnode)[0]
if not cunique_name:
continue
fname = self.__get_symbol_filename(cunique_name, cnode)
if fname != self.__default_page:
if cnode.tag == core_ns('constructor'):
filenames.insert(0, fname)
else:
filenames.append(fname)
unique_filenames = list(OrderedSet(filenames))
if not filenames:
# Did not find any symbols, trying to can get information
# about the class structure linked to that object class.
nextnode = node.getnext()
name = node.attrib['name']
if nextnode is not None and nextnode.tag == core_ns('record'):
nextnode_classfor = nextnode.attrib.get(
glib_ns('is-gtype-struct-for'))
if nextnode_classfor == name:
nunique_name = get_symbol_names(nextnode)[0]
filename = self.__get_symbol_filename(nunique_name)
if filename == self.__default_page:
self.warn(
"no-location-indication",
"No way to determine where %s should land"
" putting it to %s."
" Document the symbol for smart indexing to work" %
(unique_name, os.path.basename(filename)))
else:
filename = unique_filenames[0]
if len(unique_filenames) > 1:
self.warn(
"no-location-indication",
" Going wild here to determine where %s needs to land"
" as we could detect the following possibilities: %s." %
(unique_name, unique_filenames))
else:
self.debug(
" No class comment for %s determined that it should"
" land into %s with all other class related documentation."
% (unique_name, os.path.basename(filename)))
return filename
def parse_config(self, config):
super(GIExtension, self).parse_config(config)
ALL_GIRS.update({os.path.basename(s): s for s in self.sources})
self.c_sources = config.get_sources('gi-c')
self.source_roots = OrderedSet(config.get_paths('gi_c_source_roots'))
chosen_languages = [l.lower() for l in config.get('languages', [])]
languages = []
for lang_type in get_language_classes():
if lang_type.language_name in chosen_languages:
languages.append(lang_type())
if languages:
self.languages = languages
else:
self.languages = [
lang_type() for lang_type in get_language_classes()
]
# Make sure C always gets formatted first
c_language = self.get_language('c')
if c_language:
self.languages.remove(c_language)
self.languages.insert(0, c_language)
for gir_file in self.sources:
gir_root = etree.parse(gir_file).getroot()
cache_nodes(gir_root, ALL_GIRS, self.languages)
def __get_user_symbols(self, tree, index):
symbols = self.__get_listed_symbols_in_markdown(tree, index)
private_symbols = set()
parented_symbols = defaultdict(list)
for source_file, symbols_names in list(self._created_symbols.items()):
if source_file.endswith(('.markdown', '.md')):
continue
for symname in symbols_names:
symbol = self.app.database.get_symbol(symname)
if not symbol.parent_name:
continue
if symbol.parent_name in symbols:
symbols[symbol.unique_name] = symbols[symbol.parent_name]
else:
parented_symbols[symbol.parent_name].append(symname)
page_name = self.__get_page(tree, source_file)[1]
comment = self.__get_comment_for_page(source_file, page_name)
if not comment:
continue
for symname in comment.meta.get("private-symbols", OrderedSet()):
private_symbols.add(symname)
comment_syms, located_parented_symbols = self.__list_symbols_in_comment(
comment, parented_symbols, source_file, page_name)
if comment_syms:
comment.meta['symbols'].extend(located_parented_symbols)
symbols.update(comment_syms)
return set(symbols.keys()), set(symbols.values()), private_symbols
def __init__(self,
source_file,
ast,
output_path,
project_name,
meta=None,
raw_contents=None):
"Banana banana"
assert source_file
basename = os.path.basename(source_file)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.pre_sorted = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.private_symbols = []
self.typed_symbols = OrderedDict()
self.by_parent_symbols = OrderedDict()
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.project_name = project_name
self.cached_paths = OrderedSet()
meta = meta or {}
self.listed_symbols = []
self.symbol_names = []
self.short_description = None
self.render_subpages = True
self.title = ''
self.meta = Schema(Page.meta_schema).validate({})
self.__update_meta(meta)
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, ref)
def __resolve_patterns(self, source_patterns, from_conf):
if source_patterns is None:
return OrderedSet()
all_files = OrderedSet()
for item in source_patterns:
item = self.__abspath(item, from_conf)
if item in all_files:
continue
if not os.path.exists(item):
all_files |= glob.glob(item)
else:
all_files.add(item)
return all_files
def get_possible_path(self, name):
self.__find_package_root()
for path in OrderedSet([self.__package_root]) | self.source_roots:
possible_path = os.path.join(path, name)
if possible_path in self._get_all_sources():
return self._get_smart_filename(possible_path)
return None
def test_smart_key_custom(self):
sources = []
sources.append(
self._create_src_file('source_a.test', ['custom_key', 'symbol_1']))
self.test_ext.sources = sources
self.test_ext.use_custom_key = True
self.test_ext.setup()
self.assertDictEqual(self.test_ext.get_created_symbols(),
{'custom_key': OrderedSet(['symbol_1'])})
def __get_page(self, tree, source_file):
page_name = self.__get_rel_source_path(source_file)
for path in OrderedSet([self.__package_root]) | self.source_roots:
possible_name = os.path.relpath(source_file, path)
page = tree.get_pages().get(possible_name)
if page:
return page, page_name
return page, page_name
def __parse_config(self):
"""
Banana banana
"""
output = self.config.get_path('output') or None
self.sitemap_path = self.config.get_path('sitemap')
if self.sitemap_path is None:
error('invalid-config', 'No sitemap was provided')
if output is not None:
self.output = os.path.abspath(output)
else:
self.output = None
self.project_name = self.config.get('project_name', None)
self.project_version = self.config.get('project_version', None)
self.output_format = self.config.get('output_format')
if self.output_format not in ["html"]:
error('invalid-config',
'Unsupported output format : %s' % self.output_format)
self.__index_file = self.config.get_index()
if self.__index_file is None:
error('invalid-config', 'index is required')
if not os.path.exists(self.__index_file):
error('invalid-config',
'The provided index "%s" does not exist' % self.__index_file)
cmd_line_includes = self.config.get_paths('include_paths')
self.__base_doc_folder = os.path.dirname(self.__index_file)
self.include_paths = OrderedSet([self.__base_doc_folder])
self.include_paths |= OrderedSet(cmd_line_includes)
self.__create_change_tracker()
self.__setup_private_folder()
self.__setup_database()
self.__create_extensions()
if self.__conf_file:
self.change_tracker.add_hard_dependency(self.__conf_file)
def __init__(self, app, project):
"""Constructor for `Extension`.
This should never get called directly.
Args:
project: The `project.Project` instance which documentation
is being generated.
"""
self.project = project
self.app = app
self.sources = set()
self.smart_sources = []
self.index = None
self.source_roots = OrderedSet()
self._created_symbols = DefaultOrderedDict(OrderedSet)
self.__package_root = None
self.__toplevel_comments = OrderedSet()
self.formatter = self._make_formatter()
def __init__(self, source_file, ast, meta=None, raw_contents=None):
"Banana banana"
assert source_file
if os.path.isabs(source_file):
basename = os.path.basename(source_file)
else:
basename = source_file.replace('/', '-')
name = os.path.splitext(basename)[0]
pagename = '%s.html' % name
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, name)
def get_pagename(self, name):
self.__find_package_root()
# Find the longest prefix
longest = None
for path in OrderedSet([self.__package_root]) | self.source_roots:
commonprefix = os.path.commonprefix([path, name])
if commonprefix == path and (longest is None or len(path) > len(longest)):
longest = path
if longest is not None:
return os.path.relpath(name, longest)
return name
def setup_subpages(pagenames, get_pagename):
"""Setup subpages for pages with names in @pagenames"""
sitemap_pages = sitemap.get_all_sources()
for pagename in pagenames:
page = self.__all_pages[get_pagename(pagename)]
subpages = sitemap_pages.get(get_pagename(pagename), [])
page.subpages = OrderedSet(subpages) | page.subpages
for subpage_name in page.subpages:
if subpage_name not in unlisted_pagenames:
subpage = self.__all_pages[subpage_name]
if not subpage.meta.get('auto-sort', False):
subpage.pre_sorted = True
page.subpages -= unlisted_pagenames
def test_extension_override(self):
self.__create_md_file('source_a.test.markdown', (u'# My override\n'))
_ = self.__create_test_layout()
page = self.tree.get_pages()['source_a.test']
self.assertEqual(page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
self.assertEqual(os.path.basename(page.source_file),
'source_a.test.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(out, u'<h1>My override</h1>\n')
def __resolve_patterns(self, source_patterns, from_conf):
if source_patterns is None:
return OrderedSet()
cache_key = self.__get_key(source_patterns, from_conf)
all_files = Config.__pattern_cache.get(cache_key, OrderedSet())
if all_files:
return all_files
for item in source_patterns:
item = self.__abspath(item, from_conf)
if item in all_files:
continue
if not os.path.exists(item):
all_files |= glob.glob(item)
else:
all_files.add(item)
Config.__pattern_cache[cache_key] = all_files
return all_files