def setUp(self):
# Cruft, should be unnecessary soon
self.tag_validators = {}
self.parser = GtkDocParser(self)
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.formatter = GtkDocStringFormatter()
class TestGtkDocParser(unittest.TestCase):
def setUp(self):
# Cruft, should be unnecessary soon
self.tag_validators = {}
self.parser = GtkDocParser(self)
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.formatter = GtkDocStringFormatter()
def test_basic(self):
raw = BASIC_GTKDOC_COMMENT
lineno = 10
end_lineno = len(raw.split('\n')) + 10 - 1
comment = self.parser.parse_comment(raw, '/home/meh/test-greeter.c',
lineno, end_lineno)
self.assertEqual(comment.name, u'test_greeter_greet')
self.assertEqual(len(comment.params), 1)
self.assertTrue('greeter' in comment.params)
param = comment.params['greeter']
self.assertEqual(param.description, 'a random greeter')
def test_linenos(self):
raw = LINENOS_GTKDOC_COMMENT
lineno = 10
end_lineno = len(raw.split('\n')) + 10 - 1
comment = self.parser.parse_comment(raw, '/home/meh/test-greeter.c',
lineno, end_lineno)
self.assertEqual(comment.line_offset, 11)
param = comment.params['greeter']
self.assertEqual(param.line_offset, 5)
self.assertEqual(param.initial_col_offset, 10)
self.assertEqual(param.col_offset, 3)
def setUp(self):
# Cruft, should be unnecessary soon
self.tag_validators = {}
self.parser = GtkDocParser(self)
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.formatter = GtkDocStringFormatter()
def __init__(self, doc_repo, doc_db):
if not clang.cindex.Config.loaded:
# Let's try and find clang ourselves first
clang_libdir = get_clang_libdir()
if os.path.exists(clang_libdir):
clang.cindex.Config.set_library_path(clang_libdir)
self.__raw_comment_parser = GtkDocParser(doc_repo)
self.doc_repo = doc_repo
self.__doc_db = doc_db
class TestGtkDocParser(unittest.TestCase):
def setUp(self):
# Cruft, should be unnecessary soon
self.tag_validators = {}
self.parser = GtkDocParser(self)
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.formatter = GtkDocStringFormatter()
def test_basic(self):
raw = BASIC_GTKDOC_COMMENT
lineno = 10
end_lineno = len(raw.split('\n')) + 10 - 1
comment = self.parser.parse_comment(
raw,
'/home/meh/test-greeter.c',
lineno,
end_lineno)
self.assertEqual(comment.name, u'test_greeter_greet')
self.assertEqual(len(comment.params), 1)
self.assertTrue('greeter' in comment.params)
param = comment.params['greeter']
self.assertEqual(param.description, 'a random greeter')
def test_linenos(self):
raw = LINENOS_GTKDOC_COMMENT
lineno = 10
end_lineno = len(raw.split('\n')) + 10 - 1
comment = self.parser.parse_comment(
raw,
'/home/meh/test-greeter.c',
lineno,
end_lineno)
self.assertEqual(comment.line_offset, 11)
param = comment.params['greeter']
self.assertEqual(param.line_offset, 5)
self.assertEqual(param.initial_col_offset, 10)
self.assertEqual(param.col_offset, 3)
def __init__(self, doc_repo, ext, sources):
self.__current_filename = None
self.symbols = {}
self.doc_repo = doc_repo
self.__ext = ext
self.__raw_comment_parser = GtkDocParser(self.doc_repo)
for filename in sources:
self.__current_filename = filename
ip = InterfaceParser(filename)
for name, interface in ip.parse().iteritems():
self.__create_class_symbol (interface)
for mname, method in interface.methods.iteritems():
self.__create_function_symbol (method)
for pname, prop in interface.properties.iteritems():
self.__create_property_symbol (prop)
for sname, signal in interface.signals.iteritems():
self.__create_signal_symbol (signal)
class ClangScanner(object):
def __init__(self, doc_repo, doc_db):
if not clang.cindex.Config.loaded:
# Let's try and find clang ourselves first
clang_libdir = get_clang_libdir()
if os.path.exists(clang_libdir):
clang.cindex.Config.set_library_path(clang_libdir)
self.__raw_comment_parser = GtkDocParser(doc_repo)
self.doc_repo = doc_repo
self.__doc_db = doc_db
def scan(self, filenames, options, incremental, full_scan,
full_scan_patterns, fail_fast=False):
index = clang.cindex.Index.create()
flags = clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
info('scanning %d C source files' % len(filenames))
self.filenames = filenames
# FIXME: er maybe don't do that ?
args = ["-Wno-attributes"]
args.append ("-isystem%s" % get_clang_headers())
args.extend (options)
self.symbols = {}
self.parsed = set({})
debug('CFLAGS %s' % ' '.join(args))
header_guarded = set()
for filename in self.filenames:
if filename in self.parsed:
continue
do_full_scan = any(fnmatch(filename, p) for p in full_scan_patterns)
if do_full_scan:
debug('scanning %s' % filename)
tu = index.parse(filename, args=args, options=flags)
for diag in tu.diagnostics:
warn('clang-diagnostic', 'Clang issue : %s' % str(diag))
self.__parse_file (filename, tu, full_scan)
if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(filename))):
header_guarded.add(filename)
for include in tu.get_includes():
fname = os.path.abspath(str(include.include))
if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(fname))):
if fname in self.filenames:
header_guarded.add(fname)
self.__parse_file (fname, tu, full_scan)
if not full_scan:
for filename in filenames:
with open (filename, 'r') as f:
skip_next_symbol = filename in header_guarded
debug('Getting comments in %s' % filename)
cs = get_comments (filename)
for c in cs:
if c[4]:
block = self.__raw_comment_parser.parse_comment(c[0],
c[1], c[2], c[3], self.doc_repo.include_paths)
if block is not None:
self.doc_repo.doc_database.add_comment(block)
elif not skip_next_symbol:
if filename.endswith('.h'):
self.__create_macro_from_raw_text(c)
else:
skip_next_symbol = False
return True
def set_extension(self, extension):
self.__doc_db = extension
def __parse_file (self, filename, tu, full_scan):
if filename in self.parsed:
return
self.parsed.add (filename)
if filename not in self.filenames:
return
debug('scanning %s' % filename)
start = tu.get_location (filename, 0)
end = tu.get_location (filename, int(os.path.getsize(filename)))
extent = clang.cindex.SourceRange.from_locations (start, end)
cursors = self.__get_cursors(tu, extent)
# Happens with empty source files
if cursors is None:
return
if filename in self.filenames:
self.__create_symbols (cursors, tu)
# That's the fastest way of obtaining our ast nodes for a given filename
def __get_cursors (self, tu, extent):
tokens_memory = POINTER(clang.cindex.Token)()
tokens_count = c_uint()
clang.cindex.conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
byref(tokens_count))
count = int(tokens_count.value)
if count < 1:
return
cursors = (clang.cindex.Cursor * count)()
clang.cindex.conf.lib.clang_annotateTokens (tu, tokens_memory, tokens_count,
cursors)
return cursors
def __create_symbols(self, nodes, tu):
for node in nodes:
node._tu = tu
# This is dubious, needed to parse G_DECLARE_FINAL_TYPE
# investigate further (fortunately this doesn't seem to
# significantly impact performance ( ~ 5% )
if node.kind == clang.cindex.CursorKind.TYPE_REF:
node = node.get_definition()
if not node:
continue
if not unicode(node.location.file) in self.filenames:
continue
if node.spelling in self.symbols:
continue
sym = None
func_dec = self.__getFunctionDeclNode(node)
if func_dec and func_dec.spelling not in self.symbols:
sym = self.__create_function_symbol(func_dec)
elif node.kind == clang.cindex.CursorKind.VAR_DECL:
sym = self.__create_exported_variable_symbol (node)
elif node.kind == clang.cindex.CursorKind.TYPEDEF_DECL:
sym = self.__create_typedef_symbol (node)
elif node.kind == clang.cindex.CursorKind.STRUCT_DECL and node.spelling:
sym = self.__create_struct_symbol(node)
elif node.kind == clang.cindex.CursorKind.ENUM_DECL and node.spelling:
sym = self.__create_enum_symbol(node)
if sym is not None:
self.symbols[sym.unique_name] = sym
self.__create_symbols(node.get_children(), tu)
def __getFunctionDeclNode(self, node):
if not node.location.file:
return None
elif node.location.file.name.endswith(".h"):
if node.kind == clang.cindex.CursorKind.FUNCTION_DECL:
return node
else:
return None
if node.kind != clang.cindex.CursorKind.COMPOUND_STMT:
return None
if node.semantic_parent.kind == clang.cindex.CursorKind.FUNCTION_DECL:
return node.semantic_parent
return None
def __apply_qualifiers (self, type_, tokens):
if type_.is_const_qualified():
tokens.append ('const ')
if type_.is_restrict_qualified():
tokens.append ('restrict ')
if type_.is_volatile_qualified():
tokens.append ('volatile ')
def make_c_style_type_name (self, type_):
tokens = []
while (type_.kind == clang.cindex.TypeKind.POINTER):
self.__apply_qualifiers(type_, tokens)
tokens.append ('*')
type_ = type_.get_pointee()
if type_.kind == clang.cindex.TypeKind.TYPEDEF:
d = type_.get_declaration ()
link = Link (None, d.displayname, d.displayname)
tokens.append (link)
self.__apply_qualifiers(type_, tokens)
elif type_.kind == clang.cindex.TypeKind.UNEXPOSED:
d = type_.get_declaration()
if d.spelling:
tokens.append(Link(None, d.displayname, d.displayname))
else:
tokens.append('__UNKNOWN__')
if d.kind == clang.cindex.CursorKind.STRUCT_DECL:
tokens.append ('struct ')
elif d.kind == clang.cindex.CursorKind.ENUM_DECL:
tokens.append ('enum ')
else:
tokens.append (type_.spelling + ' ')
tokens.reverse()
return tokens
def __create_callback_symbol (self, node):
parameters = []
return_value = None
for child in node.get_children():
if not return_value:
t = node.underlying_typedef_type
res = t.get_pointee().get_result()
type_tokens = self.make_c_style_type_name (res)
return_value = [ReturnItemSymbol(type_tokens=type_tokens)]
else:
type_tokens = self.make_c_style_type_name (child.type)
parameter = ParameterSymbol (argname=child.displayname,
type_tokens=type_tokens)
parameters.append (parameter)
if not return_value:
return_value = [ReturnItemSymbol(type_tokens=[])]
sym = self.__doc_db.get_or_create_symbol(CallbackSymbol, parameters=parameters,
return_value=return_value, display_name=node.spelling,
filename=str(node.location.file), lineno=node.location.line)
return sym
def __parse_public_fields (self, decl):
tokens = decl.translation_unit.get_tokens(extent=decl.extent)
delimiters = []
filename = str(decl.location.file)
start = decl.extent.start.line
end = decl.extent.end.line + 1
original_lines = [linecache.getline(filename, i).rstrip() for i in range(start,
end)]
public = True
if (self.__locate_delimiters(tokens, delimiters)):
public = False
children = []
for child in decl.get_children():
children.append(child)
delimiters.reverse()
if not delimiters:
return '\n'.join (original_lines), children
public_children = []
children = []
for child in decl.get_children():
children.append(child)
children.reverse()
if children:
next_child = children.pop()
else:
next_child = None
next_delimiter = delimiters.pop()
final_text = []
found_first_child = False
for i, line in enumerate(original_lines):
lineno = i + start
if next_delimiter and lineno == next_delimiter[1]:
public = next_delimiter[0]
if delimiters:
next_delimiter = delimiters.pop()
else:
next_delimiter = None
continue
if not next_child or lineno < next_child.location.line:
if public or not found_first_child:
final_text.append (line)
continue
if lineno == next_child.location.line:
found_first_child = True
if public:
final_text.append (line)
public_children.append (next_child)
while next_child.location.line == lineno:
if not children:
public = True
next_child = None
break
next_child = children.pop()
return ('\n'.join(final_text), public_children)
def __locate_delimiters (self, tokens, delimiters):
public_pattern = "/*<public>*/"
private_pattern = "/*<private>*/"
protected_pattern = "/*<protected>*/"
had_public = False
for tok in tokens:
if tok.kind == clang.cindex.TokenKind.COMMENT:
comment = ''.join(tok.spelling.split())
if public_pattern == comment:
had_public = True
delimiters.append((True, tok.location.line))
elif private_pattern == comment:
delimiters.append((False, tok.location.line))
elif protected_pattern == comment:
delimiters.append((False, tok.location.line))
return had_public
def __create_struct_symbol (self, node, spelling=None):
spelling = spelling or node.spelling
raw_text, public_fields = self.__parse_public_fields (node)
raw_text = unicode(raw_text.decode('utf8'))
members = []
for field in public_fields:
type_tokens = self.make_c_style_type_name (field.type)
is_function_pointer = ast_node_is_function_pointer (field.type)
qtype = QualifiedSymbol(type_tokens=type_tokens)
name = '%s.%s' % (spelling, field.spelling)
member = FieldSymbol (is_function_pointer=is_function_pointer,
member_name=field.spelling, qtype=qtype,
display_name=name, unique_name=name)
members.append (member)
if not public_fields:
raw_text = None
anonymous = not node.spelling
return self.__doc_db.get_or_create_symbol(StructSymbol, raw_text=raw_text,
members=members, anonymous=anonymous,
display_name=spelling,
filename=str(node.location.file), lineno=node.location.line)
def __create_enum_symbol (self, node, spelling=None):
spelling = spelling or node.spelling
members = []
for member in node.get_children():
member_value = member.enum_value
# FIXME: this is pretty much a macro symbol ?
member = self.__doc_db.get_or_create_symbol(Symbol, display_name=member.spelling,
filename=str(member.location.file),
lineno=member.location.line)
member.enum_value = member_value
members.append (member)
anonymous = not node.spelling
start = node.extent.start.line
end = node.extent.end.line + 1
original_lines = [linecache.getline(str(node.location.file), i).rstrip() for i in range(start,
end)]
raw_text = '\n'.join(original_lines)
return self.__doc_db.get_or_create_symbol(EnumSymbol, members=members,
anonymous=anonymous, raw_text=raw_text, display_name=spelling,
filename=str(node.location.file), lineno=node.location.line)
def __create_alias_symbol (self, node):
type_tokens = self.make_c_style_type_name(node.underlying_typedef_type)
aliased_type = QualifiedSymbol (type_tokens=type_tokens)
return self.__doc_db.get_or_create_symbol(AliasSymbol, aliased_type=aliased_type,
display_name=node.spelling, filename=str(node.location.file),
lineno=node.location.line)
def __create_typedef_symbol (self, node):
t = node.underlying_typedef_type
decl = t.get_declaration()
if ast_node_is_function_pointer (t):
sym = self.__create_callback_symbol (node)
elif not decl.spelling and decl.kind == clang.cindex.CursorKind.STRUCT_DECL: # typedef struct {} foo;
sym = self.__create_struct_symbol (decl, spelling=node.spelling)
elif not decl.spelling and decl.kind == clang.cindex.CursorKind.ENUM_DECL: # typedef enum {} bar;
sym = self.__create_enum_symbol (decl, spelling=node.spelling)
else:
sym = self.__create_alias_symbol (node)
return sym
def __create_macro_from_raw_text(self, raw):
mcontent = raw[0].replace('\t', ' ')
mcontent = mcontent.split(' ', 1)[1]
split = mcontent.split('(', 1)
name = split[0]
if not (' ' in name or '\t' in name) and len(split) == 2:
args = split[1].split(')', 1)[0].split(',')
if args:
stripped_name = name.strip()
return self.__create_function_macro_symbol(stripped_name,
raw[1], raw[2], raw[0])
name = mcontent.split(' ', 1)[0]
stripped_name = name.strip()
return self.__create_constant_symbol(stripped_name, raw[1], raw[2], raw[0])
def __create_function_macro_symbol (self, name, filename, lineno, original_text):
comment = self.doc_repo.doc_database.get_comment(name)
return_value = [None]
if comment:
return_tag = comment.tags.get ('returns')
if return_tag:
return_value = [ReturnItemSymbol ()]
parameters = []
if comment:
for param_name in comment.params:
parameter = ParameterSymbol (argname=param_name)
parameters.append (parameter)
sym = self.__doc_db.get_or_create_symbol(FunctionMacroSymbol, return_value=return_value,
parameters=parameters, original_text=original_text,
display_name=name,
filename=filename, lineno=lineno)
return sym
def __create_constant_symbol (self, name, filename, lineno, original_text):
return self.__doc_db.get_or_create_symbol(ConstantSymbol,
original_text=original_text,
display_name=name, filename=filename,
lineno=lineno)
def __create_function_symbol (self, node):
parameters = []
type_tokens = self.make_c_style_type_name (node.result_type)
return_value = [ReturnItemSymbol (type_tokens=type_tokens)]
for param in node.get_arguments():
type_tokens = self.make_c_style_type_name (param.type)
parameter = ParameterSymbol (argname=param.displayname,
type_tokens=type_tokens)
parameters.append (parameter)
sym = self.__doc_db.get_or_create_symbol(FunctionSymbol, parameters=parameters,
return_value=return_value, display_name=node.spelling,
filename=str(node.location.file), lineno=node.location.line,
extent_start=node.extent.start.line,
extent_end=node.extent.end.line)
return sym
def __create_exported_variable_symbol (self, node):
l = linecache.getline (str(node.location.file), node.location.line)
split = l.split()
start = node.extent.start.line
end = node.extent.end.line + 1
filename = str(node.location.file)
original_lines = [linecache.getline(filename, i).rstrip() for i in range(start,
end)]
original_text = '\n'.join(original_lines)
type_tokens = self.make_c_style_type_name(node.type)
type_qs = QualifiedSymbol(type_tokens=type_tokens)
sym = self.__doc_db.get_or_create_symbol(ExportedVariableSymbol, original_text=original_text,
display_name=node.spelling, filename=str(node.location.file),
lineno=node.location.line, type_qs=type_qs)
return sym
class DBusScanner(object):
def __init__(self, doc_repo, ext, sources):
self.__current_filename = None
self.symbols = {}
self.doc_repo = doc_repo
self.__ext = ext
self.__raw_comment_parser = GtkDocParser(self.doc_repo)
for filename in sources:
self.__current_filename = filename
ip = InterfaceParser(filename)
for name, interface in ip.parse().iteritems():
self.__create_class_symbol (interface)
for mname, method in interface.methods.iteritems():
self.__create_function_symbol (method)
for pname, prop in interface.properties.iteritems():
self.__create_property_symbol (prop)
for sname, signal in interface.signals.iteritems():
self.__create_signal_symbol (signal)
def __create_parameters (self, nodes, omit_direction=False):
parameters = []
for param in nodes:
type_tokens = []
if not omit_direction:
type_tokens.append (param.direction.upper() + ' ')
type_tokens.append (param.type)
parameters.append (ParameterSymbol (argname=param.name,
type_tokens=type_tokens))
return parameters
def __comment_from_node(self, node, unique_name=None):
if node.comment is None:
return None
lineno = -1
lines = node.comment.split('\n')
stripped_lines = []
column_offset = 0
line_offset = 0
for l in lines:
nl = l.strip()
if not nl and not stripped_lines:
line_offset += 1
continue
if not column_offset and nl:
column_offset = len(l) - len(nl)
stripped_lines.append(nl)
if hasattr(node, 'comment_lineno'):
lineno = node.comment_lineno + line_offset
comment = u'\n'.join(stripped_lines)
comment = self.__raw_comment_parser.parse_comment (comment,
self.__current_filename, lineno,
-1, stripped=True)
if comment:
comment.col_offset = column_offset + 1
for param in comment.params.values():
param.col_offset = comment.col_offset
if unique_name and comment:
comment.name = unique_name
return comment
def __create_function_symbol (self, node):
unique_name = '%s.%s' % (self.__current_class_name, node.name)
comment = self.__comment_from_node(node, unique_name)
self.doc_repo.doc_database.add_comment(comment)
parameters = self.__create_parameters (node.arguments)
self.__ext.get_or_create_symbol(FunctionSymbol,
parameters=parameters,
display_name=node.name,
filename=self.__current_filename,
unique_name=unique_name)
def __create_class_symbol (self, node):
self.__current_class_name = node.name
comment = self.__comment_from_node(node, node.name)
self.doc_repo.doc_database.add_comment(comment)
self.__ext.get_or_create_symbol(ClassSymbol,
display_name=node.name,
filename=self.__current_filename)
def __create_property_symbol (self, node):
unique_name = '%s.%s' % (self.__current_class_name, node.name)
comment = self.__comment_from_node(node, unique_name)
self.doc_repo.doc_database.add_comment(comment)
type_tokens = [node.type]
type_ = QualifiedSymbol (type_tokens=type_tokens)
flags = ''
if node.access == node.ACCESS_READ:
flags = 'Read'
elif node.access == node.ACCESS_WRITE:
flags = 'Write'
elif node.access == node.ACCESS_READWRITE:
flags = 'Read / Write'
sym = self.__ext.get_or_create_symbol(PropertySymbol,
prop_type=type_,
display_name=node.name,
unique_name=unique_name,
filename=self.__current_filename)
if flags:
sym.extension_contents['Flags'] = flags
def __create_signal_symbol (self, node):
unique_name = '%s.%s' % (self.__current_class_name, node.name)
comment = self.__comment_from_node(node, unique_name)
self.doc_repo.doc_database.add_comment(comment)
parameters = self.__create_parameters (node.arguments,
omit_direction=True)
self.__ext.get_or_create_symbol(SignalSymbol,
parameters=parameters,
display_name=node.name, unique_name=unique_name,
filename=self.__current_filename)
