def test_source_relocated_twice(self):
sitemap = (u'index.markdown\n'
'\ttest-index\n')
index_path = self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
a_source_path = 'source_a.test'
b_source_path = 'source_b.test'
c_source_path = 'source_c.test'
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': a_source_path,
}
),
]
comments = [
Comment(name='page_1',
meta={'sources': ['source_a.test']},
filename=os.path.join(self.__src_dir, b_source_path),
toplevel=True),
Comment(name='page_1',
meta={'sources': ['source_a.test']},
filename=os.path.join(self.__src_dir, c_source_path),
toplevel=True),
]
with self.assertRaises(InvalidRelocatedSourceException):
self.__make_project(sitemap, index_path, symbols=symbols, comments=comments,
output=self.__output_dir)
def test_symbol_marked_private_twice(self):
sitemap = (u'index.markdown\n'
'\ttest-index\n')
index_path = self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
a_source_path = 'source_a.test'
b_source_path = 'source_b.test'
c_source_path = 'source_c.test'
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': a_source_path,
}
),
]
comments = [
Comment(name='page_1',
meta={'private-symbols': ['symbol_a']},
filename=b_source_path,
toplevel=True),
Comment(name='page_2',
meta={'private-symbols': ['symbol_a']},
filename=c_source_path,
toplevel=True),
]
with self.assertRaises(SymbolListedTwiceException):
self.__make_project(sitemap, index_path, symbols=symbols, comments=comments,
output=self.__output_dir)
def __extract_feature_comment(self, feature_type, feature):
pagename = feature_type + '-' + feature['name']
possible_comment_names = [pagename, feature['name']]
if feature_type == 'element':
possible_comment_names.append(feature['hierarchy'][0])
comment = None
for comment_name in possible_comment_names:
comment = self.app.database.get_comment(comment_name)
if comment:
break
description = feature.get('description')
if not comment:
comment = Comment(
pagename,
Comment(description=feature['name']),
description=description,
short_description=Comment(description=description))
self.app.database.add_comment(comment)
elif not comment.short_description:
comment.short_description = Comment(description=description)
comment.title = Comment(description=feature['name'])
comment.name = feature.get('name', pagename)
comment.meta['title'] = feature['name']
self.__toplevel_comments.add(comment)
return pagename, comment
def test_multiple_toplevel_comments(self):
sitemap = ('index.markdown\n'
'\ttest-index\n')
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': 'source1.test'
}
),
(
[FunctionSymbol],
{
'unique_name': 'symbol_b',
'filename': 'source1.test'
}
),
(
[FunctionSymbol],
{
'unique_name': 'symbol_c',
'filename': 'source1.test'
}
),
]
comments = [
Comment(name='foo',
filename=os.path.join(self.__src_dir, 'source1.test'),
meta={'symbols': ['symbol_a']}, toplevel=True),
Comment(name='bar',
filename=os.path.join(self.__src_dir, 'source1.test'),
meta={'symbols': ['symbol_b']}, toplevel=True),
]
self.__create_test_layout(
sitemap=sitemap, symbols=symbols, comments=comments,
output=self.__output_dir)
pages = self.app.project.tree.get_pages()
self.assertEqual(len(pages), 4)
self.assertIn('foo', pages)
foo = pages['foo']
self.assertEqual(foo.symbol_names, ['symbol_a', 'symbol_c'])
self.assertIn('bar', pages)
foo = pages['bar']
self.assertEqual(foo.symbol_names, ['symbol_b'])
def test_section_and_path_conflict(self):
sitemap = (u'index.markdown\n'
'\ttest-index\n')
index_path = self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
a_source_path = 'source1.test'
b_source_path = 'source2.test'
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': a_source_path,
}
),
]
comments = [
Comment(name='source1',
filename=b_source_path,
meta={'auto-sort': True}, toplevel=True),
]
with self.assertRaises(InvalidOutputException):
self.__make_project(sitemap, index_path, symbols=symbols, comments=comments,
output=self.__output_dir)
def __create_vfunc_symbol(self, node, parent_name):
klass_node = node.getparent()
ns = klass_node.getparent()
gtype_struct = klass_node.attrib.get(glib_ns('type-struct'))
klass_comment = self.app.database.get_comment(
'%s%s' % (ns.attrib['name'], gtype_struct))
unique_name, name, klass_name = get_symbol_names(node)
# Virtual methods are documented in the class comment
if klass_comment:
param_comment = klass_comment.params.get(name)
if (param_comment):
self.app.database.add_comment(
Comment(name=unique_name,
meta={'description': param_comment.description},
annotations=param_comment.annotations))
parameters, retval = self.__create_parameters_and_retval(node)
symbol = self.get_or_create_symbol(
VFunctionSymbol,
node,
parameters=parameters,
return_value=retval,
display_name=name,
unique_name=unique_name,
filename=self.__get_symbol_filename(klass_name),
parent_name=parent_name,
aliases=[unique_name.replace('::', '.')])
self.__sort_parameters(symbol, retval, parameters)
return symbol
def __fetch_comment(self, sym, database):
sym.comment = database.get_comment(
sym.unique_name) or Comment(sym.unique_name)
for sym in sym.get_children_symbols():
if isinstance(sym, Symbol):
self.__fetch_comment(sym, database)
def __parse_parameter(self, name, desc):
name = name.strip()[1:-1].strip()
desc = desc.strip()
desc, annotations = self.__extract_annotations(desc)
annotations = {
annotation.name: annotation
for annotation in annotations
}
return Comment(name=name, annotations=annotations, description=desc)
def __parse_parameter(self, name, desc):
name = name.strip()[1:-1].strip()
raw_comment = '%s:%s' % (name, desc)
desc = desc.strip()
desc, annotations = self.__extract_annotations(desc)
annotations = {annotation.name: annotation for annotation in
annotations}
return Comment(name=name, annotations=annotations,
meta={'description': desc}, raw_comment=raw_comment)
def __fetch_comment(self, sym, database):
old_comment = sym.comment
new_comment = database.get_comment(sym.unique_name)
sym.comment = Comment(sym.unique_name)
if new_comment:
sym.comment = new_comment
elif old_comment:
if old_comment.filename not in ChangeTracker.all_stale_files:
sym.comment = old_comment
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
self.typed_symbols = self.__get_empty_typed_symbols()
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
if self.meta.get("auto-sort", True):
all_syms = sorted(all_syms, key=lambda x: x.unique_name)
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
# Always put symbols with no parent at the end
no_parent_syms = self.by_parent_symbols.pop(None, None)
if no_parent_syms:
self.by_parent_symbols[None] = no_parent_syms
for sym_type in [
ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.name)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
def test_extension_auto_sorted_override(self):
sitemap = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_b.test\n'
'\t\t\tsource_a.test\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
comments = [
Comment(name='source_b.test',
filename=os.path.join(self.__src_dir, 'source_b.test'),
meta={'auto-sort': True}, toplevel=True),
]
self.__create_test_layout(sitemap=sitemap, comments=comments)
pages = self.app.project.tree.get_pages()
self.assertTrue(pages['source_a.test'].pre_sorted)
self.assertFalse(pages['source_b.test'].pre_sorted)
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
typed_symbols_list = namedtuple('TypedSymbolsList',
['name', 'symbols'])
for subclass in all_subclasses(Symbol):
self.typed_symbols[subclass] = typed_symbols_list(
subclass.get_plural_name(), [])
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
for sym_type in [
ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.source_file)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
def test_comment_relocation_basic(self):
sitemap = ('index.markdown\n'
'\ttest-index\n')
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': 'source1.test'
}
),
(
[FunctionSymbol],
{
'unique_name': 'symbol_b',
'filename': 'source2.test'
}
),
]
# symbol_b should be documented in source1
comments = [
Comment(name='source1.test',
filename=os.path.join(self.__src_dir, 'source1.test'),
meta={'symbols': ['symbol_b']}, toplevel=True),
]
self.__create_test_layout(
sitemap=sitemap, symbols=symbols, comments=comments,
output=self.__output_dir,
source_roots=[os.path.join(self.__src_dir, 'a'),
os.path.join(self.__src_dir, 'b')])
pages = self.app.project.tree.get_pages()
self.assertEqual(len(pages), 3)
self.assertIn('source1.test', pages)
source1 = self.app.project.tree.get_pages()['source1.test']
self.assertEqual(source1.symbol_names, ['symbol_a', 'symbol_b'])
# source2 should not appear in the sitemap
self.assertNotIn('source2.test', pages)
def test_relocation_from_source(self):
sitemap = ('index.markdown\n'
'\ttest-index\n')
symbols = [
(
[FunctionSymbol],
{
'unique_name': 'symbol_a',
'filename': os.path.join('a', 'source1.test')
}
),
(
[FunctionSymbol],
{
'unique_name': 'symbol_b',
'filename': os.path.join('b', 'source2.test')
}
),
]
# symbol_b should be documented in page_1
comments = [
Comment(name='page_1',
filename=os.path.join(self.__src_dir, 'a', 'source1.test'),
meta={'sources': [os.path.join(os.pardir, 'b', 'source2.test')]}, toplevel=True),
]
self.__create_test_layout(
sitemap=sitemap, symbols=symbols, comments=comments,
output=self.__output_dir)
pages = self.app.project.tree.get_pages()
self.assertIn('page_1', pages)
source1 = self.app.project.tree.get_pages()['page_1']
self.assertEqual(source1.symbol_names, ['symbol_a', 'symbol_b'])
# source2 should not appear in the with only its section
self.assertNotIn(os.path.join('b', 'source2.test'), pages)
self.assertEqual(len(pages), 3)
def __create_property_symbols(self,
obj,
parent_uniquename,
pagename,
parent_name=None):
properties = obj.get('properties', [])
if not properties:
return
gi_extension = self.project.extensions.get('gi-extension')
python_lang = gi_extension.get_language('python')
for name, prop in properties.items():
unique_name = '%s:%s' % (obj.get('name', parent_uniquename), name)
flags = [ReadableFlag()]
if prop['writable']:
flags += [WritableFlag()]
if prop['construct-only']:
flags += [ConstructOnlyFlag()]
elif prop['construct']:
flags += [ConstructFlag()]
type_name = prop['type-name']
tokens = type_tokens_from_type_name(type_name, python_lang)
type_ = QualifiedSymbol(type_tokens=tokens)
default = prop.get('default')
enum = prop.get('values')
if enum:
type_ = self.__create_enum_symbol(prop['type-name'],
enum,
obj.get(
'name',
parent_uniquename),
parent_name=parent_name)
if obj['hierarchy'][0] != parent_uniquename:
aliases = self._get_aliases(
['%s:%s' % (obj['hierarchy'][0], name)])
else:
aliases = []
res = self.app.database.get_symbol(unique_name)
if res is None:
res = self.create_symbol(
PropertySymbol,
prop_type=type_,
display_name=name,
unique_name=unique_name,
aliases=aliases,
parent_name=parent_name,
extra={'gst-element-name': pagename},
)
assert res
if not self.app.database.get_comment(unique_name):
comment = Comment(unique_name,
Comment(name=name),
description=prop['blurb'])
self.app.database.add_comment(comment)
# FIXME This is incorrect, it's not yet format time (from gi_extension)
extra_content = self.formatter.format_flags(flags)
res.extension_contents['Flags'] = extra_content
if default:
if prop['type-name'] in ['GstCaps', 'GstStructure']:
default = '<pre class="language-yaml">' + \
'<code class="language-yaml">%s</code></pre>' % default
res.extension_contents['Default value'] = default
def parse_comment(self, comment, filename, lineno, endlineno,
include_paths=None, stripped=False):
"""
Returns a Comment given a string
"""
if not stripped and not self.__validate_c_comment(comment.strip()):
return None
title_offset = 0
column_offset = 0
raw_comment = comment
if not stripped:
try:
while comment[column_offset * -1 - 1] != '\n':
column_offset += 1
except IndexError:
column_offset = 0
comment, title_offset = self.__strip_comment(comment)
title_and_params, description = self.__extract_titles_params_and_description(comment)
try:
block_name, parameters, annotations, is_section = \
self.__parse_title_and_parameters(filename, title_and_params)
except HotdocSourceException as _:
warn('gtk-doc-bad-syntax',
message=_.message,
filename=filename,
lineno=lineno + title_offset)
return None
params_offset = 0
for param in parameters:
param.filename = filename
param.lineno = lineno
param_offset = param.line_offset
param.line_offset = title_offset + params_offset + 1
params_offset += param_offset
param.col_offset = column_offset
if not block_name:
return None
description_offset = 0
meta = {}
tags = []
if description is not None:
n_lines = len(comment.split('\n'))
description_offset = (title_offset + n_lines -
len(description.split('\n')))
meta['description'], tags = self.__parse_description_and_tags(description)
actual_parameters = OrderedDict({})
for param in parameters:
if is_section:
cleaned_up_name = param.name.lower().replace('_', '-')
if cleaned_up_name in ['symbols', 'private-symbols', 'auto-sort', 'sources']:
meta.update(self.__parse_yaml_comment(param, filename))
if cleaned_up_name == 'sources':
sources_paths = [os.path.abspath(os.path.join(os.path.dirname(filename), path)) for path in meta[cleaned_up_name]]
meta[cleaned_up_name] = sources_paths
else:
meta[param.name] = param.description
else:
actual_parameters[param.name] = param
annotations = {annotation.name: annotation for annotation in
annotations}
tags = {tag.name.lower(): tag for tag in tags}
block = Comment(name=block_name, filename=filename, lineno=lineno,
endlineno=endlineno,
annotations=annotations, params=actual_parameters,
tags=tags, raw_comment=raw_comment,
meta=meta, toplevel=is_section)
block.line_offset = description_offset
block.col_offset = column_offset
return block
def parse_comment(self,
comment,
filename,
lineno,
endlineno,
include_paths=None,
stripped=False):
"""
Returns a Comment given a string
"""
if not stripped and not self.__validate_c_comment(comment.strip()):
return None
title_offset = 0
column_offset = 0
raw_comment = comment
if not stripped:
try:
while comment[column_offset * -1 - 1] != '\n':
column_offset += 1
except IndexError:
column_offset = 0
comment, title_offset = self.__strip_comment(comment)
split = re.split(r'\n[\W]*\n', comment, maxsplit=1)
try:
block_name, parameters, annotations, is_section = \
self.__parse_title_and_parameters(filename, split[0])
except HotdocSourceException as _:
warn('gtk-doc-bad-syntax',
message=_.message,
filename=filename,
lineno=lineno + title_offset)
return None
params_offset = 0
for param in parameters:
param.filename = filename
param.lineno = lineno
param_offset = param.line_offset
param.line_offset = title_offset + params_offset + 1
params_offset += param_offset
param.col_offset = column_offset
if not block_name:
return None
description_offset = 0
meta = {}
tags = []
if len(split) > 1:
n_lines = len(comment.split('\n'))
description_offset = (title_offset + n_lines -
len(split[1].split('\n')))
meta['description'], tags = self.__parse_description_and_tags(
split[1])
actual_parameters = OrderedDict({})
for param in parameters:
if is_section:
if param.name.lower().replace('_', '-') in [
'symbols', 'private-symbols', 'auto-sort'
]:
meta.update(self.__parse_yaml_comment(param, filename))
else:
meta[param.name] = param.description
else:
actual_parameters[param.name] = param
annotations = {
annotation.name: annotation
for annotation in annotations
}
tags = {tag.name.lower(): tag for tag in tags}
block = Comment(name=block_name,
filename=filename,
lineno=lineno,
endlineno=endlineno,
annotations=annotations,
params=actual_parameters,
tags=tags,
raw_comment=raw_comment,
meta=meta)
block.line_offset = description_offset
block.col_offset = column_offset
return block
def parse_comment(self,
comment,
filename,
lineno,
endlineno,
include_paths=None,
stripped=False):
"""
Returns a Comment given a string
"""
if not stripped and not self.__validate_c_comment(comment.strip()):
return None
title_offset = 0
column_offset = 0
raw_comment = comment
if not stripped:
try:
while comment[column_offset * -1 - 1] != '\n':
column_offset += 1
except IndexError:
column_offset = 0
comment, title_offset = self.__strip_comment(comment)
split = re.split(r'\n[\W]*\n', comment, maxsplit=1)
try:
block_name, parameters, annotations = \
self.__parse_title_and_parameters(filename, split[0])
except HotdocSourceException as _:
warn('gtk-doc-bad-syntax',
message=_.message,
filename=filename,
lineno=lineno + title_offset)
return None
params_offset = 0
for param in parameters:
param.filename = filename
param.lineno = lineno
param_offset = param.line_offset
param.line_offset = title_offset + params_offset + 1
params_offset += param_offset
param.col_offset = column_offset
if not block_name:
return None
description_offset = 0
description = ""
tags = []
if len(split) > 1:
n_lines = len(comment.split('\n'))
description_offset = (title_offset + n_lines -
len(split[1].split('\n')))
description, tags = self.__parse_description_and_tags(split[1])
title = None
short_description = None
actual_parameters = OrderedDict({})
for param in parameters:
if param.name.lower() == 'short_description':
short_description = param
elif param.name.lower() == 'title':
title = param
else:
actual_parameters[param.name] = param
annotations = {
annotation.name: annotation
for annotation in annotations
}
tags = {tag.name.lower(): tag for tag in tags}
block = Comment(name=block_name,
filename=filename,
lineno=lineno,
endlineno=endlineno,
annotations=annotations,
params=actual_parameters,
description=description,
short_description=short_description,
title=title,
tags=tags,
raw_comment=raw_comment)
block.line_offset = description_offset
block.col_offset = column_offset
return block