def setUp(self): # Cruft, should be unnecessary soon self.tag_validators = {} self.parser = GtkDocParser(self) self.doc_database = DocDatabase() self.link_resolver = LinkResolver(self.doc_database) self.formatter = GtkDocStringFormatter()
class TestGtkDocParser(unittest.TestCase): def setUp(self): # Cruft, should be unnecessary soon self.tag_validators = {} self.parser = GtkDocParser(self) self.doc_database = DocDatabase() self.link_resolver = LinkResolver(self.doc_database) self.formatter = GtkDocStringFormatter() def test_basic(self): raw = BASIC_GTKDOC_COMMENT lineno = 10 end_lineno = len(raw.split('\n')) + 10 - 1 comment = self.parser.parse_comment(raw, '/home/meh/test-greeter.c', lineno, end_lineno) self.assertEqual(comment.name, u'test_greeter_greet') self.assertEqual(len(comment.params), 1) self.assertTrue('greeter' in comment.params) param = comment.params['greeter'] self.assertEqual(param.description, 'a random greeter') def test_linenos(self): raw = LINENOS_GTKDOC_COMMENT lineno = 10 end_lineno = len(raw.split('\n')) + 10 - 1 comment = self.parser.parse_comment(raw, '/home/meh/test-greeter.c', lineno, end_lineno) self.assertEqual(comment.line_offset, 11) param = comment.params['greeter'] self.assertEqual(param.line_offset, 5) self.assertEqual(param.initial_col_offset, 10) self.assertEqual(param.col_offset, 3)
def __init__(self, doc_repo, doc_db): if not clang.cindex.Config.loaded: # Let's try and find clang ourselves first clang_libdir = get_clang_libdir() if os.path.exists(clang_libdir): clang.cindex.Config.set_library_path(clang_libdir) self.__raw_comment_parser = GtkDocParser(doc_repo) self.doc_repo = doc_repo self.__doc_db = doc_db
class TestGtkDocParser(unittest.TestCase): def setUp(self): # Cruft, should be unnecessary soon self.tag_validators = {} self.parser = GtkDocParser(self) self.doc_database = DocDatabase() self.link_resolver = LinkResolver(self.doc_database) self.formatter = GtkDocStringFormatter() def test_basic(self): raw = BASIC_GTKDOC_COMMENT lineno = 10 end_lineno = len(raw.split('\n')) + 10 - 1 comment = self.parser.parse_comment( raw, '/home/meh/test-greeter.c', lineno, end_lineno) self.assertEqual(comment.name, u'test_greeter_greet') self.assertEqual(len(comment.params), 1) self.assertTrue('greeter' in comment.params) param = comment.params['greeter'] self.assertEqual(param.description, 'a random greeter') def test_linenos(self): raw = LINENOS_GTKDOC_COMMENT lineno = 10 end_lineno = len(raw.split('\n')) + 10 - 1 comment = self.parser.parse_comment( raw, '/home/meh/test-greeter.c', lineno, end_lineno) self.assertEqual(comment.line_offset, 11) param = comment.params['greeter'] self.assertEqual(param.line_offset, 5) self.assertEqual(param.initial_col_offset, 10) self.assertEqual(param.col_offset, 3)
def __init__(self, doc_repo, ext, sources): self.__current_filename = None self.symbols = {} self.doc_repo = doc_repo self.__ext = ext self.__raw_comment_parser = GtkDocParser(self.doc_repo) for filename in sources: self.__current_filename = filename ip = InterfaceParser(filename) for name, interface in ip.parse().iteritems(): self.__create_class_symbol (interface) for mname, method in interface.methods.iteritems(): self.__create_function_symbol (method) for pname, prop in interface.properties.iteritems(): self.__create_property_symbol (prop) for sname, signal in interface.signals.iteritems(): self.__create_signal_symbol (signal)
class ClangScanner(object): def __init__(self, doc_repo, doc_db): if not clang.cindex.Config.loaded: # Let's try and find clang ourselves first clang_libdir = get_clang_libdir() if os.path.exists(clang_libdir): clang.cindex.Config.set_library_path(clang_libdir) self.__raw_comment_parser = GtkDocParser(doc_repo) self.doc_repo = doc_repo self.__doc_db = doc_db def scan(self, filenames, options, incremental, full_scan, full_scan_patterns, fail_fast=False): index = clang.cindex.Index.create() flags = clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD info('scanning %d C source files' % len(filenames)) self.filenames = filenames # FIXME: er maybe don't do that ? args = ["-Wno-attributes"] args.append ("-isystem%s" % get_clang_headers()) args.extend (options) self.symbols = {} self.parsed = set({}) debug('CFLAGS %s' % ' '.join(args)) header_guarded = set() for filename in self.filenames: if filename in self.parsed: continue do_full_scan = any(fnmatch(filename, p) for p in full_scan_patterns) if do_full_scan: debug('scanning %s' % filename) tu = index.parse(filename, args=args, options=flags) for diag in tu.diagnostics: warn('clang-diagnostic', 'Clang issue : %s' % str(diag)) self.__parse_file (filename, tu, full_scan) if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(filename))): header_guarded.add(filename) for include in tu.get_includes(): fname = os.path.abspath(str(include.include)) if (clang.cindex.conf.lib.clang_isFileMultipleIncludeGuarded(tu, tu.get_file(fname))): if fname in self.filenames: header_guarded.add(fname) self.__parse_file (fname, tu, full_scan) if not full_scan: for filename in filenames: with open (filename, 'r') as f: skip_next_symbol = filename in header_guarded debug('Getting comments in %s' % filename) cs = get_comments (filename) for c in cs: if c[4]: block = self.__raw_comment_parser.parse_comment(c[0], c[1], c[2], c[3], self.doc_repo.include_paths) if block is not None: self.doc_repo.doc_database.add_comment(block) elif not skip_next_symbol: if filename.endswith('.h'): self.__create_macro_from_raw_text(c) else: skip_next_symbol = False return True def set_extension(self, extension): self.__doc_db = extension def __parse_file (self, filename, tu, full_scan): if filename in self.parsed: return self.parsed.add (filename) if filename not in self.filenames: return debug('scanning %s' % filename) start = tu.get_location (filename, 0) end = tu.get_location (filename, int(os.path.getsize(filename))) extent = clang.cindex.SourceRange.from_locations (start, end) cursors = self.__get_cursors(tu, extent) # Happens with empty source files if cursors is None: return if filename in self.filenames: self.__create_symbols (cursors, tu) # That's the fastest way of obtaining our ast nodes for a given filename def __get_cursors (self, tu, extent): tokens_memory = POINTER(clang.cindex.Token)() tokens_count = c_uint() clang.cindex.conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), byref(tokens_count)) count = int(tokens_count.value) if count < 1: return cursors = (clang.cindex.Cursor * count)() clang.cindex.conf.lib.clang_annotateTokens (tu, tokens_memory, tokens_count, cursors) return cursors def __create_symbols(self, nodes, tu): for node in nodes: node._tu = tu # This is dubious, needed to parse G_DECLARE_FINAL_TYPE # investigate further (fortunately this doesn't seem to # significantly impact performance ( ~ 5% ) if node.kind == clang.cindex.CursorKind.TYPE_REF: node = node.get_definition() if not node: continue if not unicode(node.location.file) in self.filenames: continue if node.spelling in self.symbols: continue sym = None func_dec = self.__getFunctionDeclNode(node) if func_dec and func_dec.spelling not in self.symbols: sym = self.__create_function_symbol(func_dec) elif node.kind == clang.cindex.CursorKind.VAR_DECL: sym = self.__create_exported_variable_symbol (node) elif node.kind == clang.cindex.CursorKind.TYPEDEF_DECL: sym = self.__create_typedef_symbol (node) elif node.kind == clang.cindex.CursorKind.STRUCT_DECL and node.spelling: sym = self.__create_struct_symbol(node) elif node.kind == clang.cindex.CursorKind.ENUM_DECL and node.spelling: sym = self.__create_enum_symbol(node) if sym is not None: self.symbols[sym.unique_name] = sym self.__create_symbols(node.get_children(), tu) def __getFunctionDeclNode(self, node): if not node.location.file: return None elif node.location.file.name.endswith(".h"): if node.kind == clang.cindex.CursorKind.FUNCTION_DECL: return node else: return None if node.kind != clang.cindex.CursorKind.COMPOUND_STMT: return None if node.semantic_parent.kind == clang.cindex.CursorKind.FUNCTION_DECL: return node.semantic_parent return None def __apply_qualifiers (self, type_, tokens): if type_.is_const_qualified(): tokens.append ('const ') if type_.is_restrict_qualified(): tokens.append ('restrict ') if type_.is_volatile_qualified(): tokens.append ('volatile ') def make_c_style_type_name (self, type_): tokens = [] while (type_.kind == clang.cindex.TypeKind.POINTER): self.__apply_qualifiers(type_, tokens) tokens.append ('*') type_ = type_.get_pointee() if type_.kind == clang.cindex.TypeKind.TYPEDEF: d = type_.get_declaration () link = Link (None, d.displayname, d.displayname) tokens.append (link) self.__apply_qualifiers(type_, tokens) elif type_.kind == clang.cindex.TypeKind.UNEXPOSED: d = type_.get_declaration() if d.spelling: tokens.append(Link(None, d.displayname, d.displayname)) else: tokens.append('__UNKNOWN__') if d.kind == clang.cindex.CursorKind.STRUCT_DECL: tokens.append ('struct ') elif d.kind == clang.cindex.CursorKind.ENUM_DECL: tokens.append ('enum ') else: tokens.append (type_.spelling + ' ') tokens.reverse() return tokens def __create_callback_symbol (self, node): parameters = [] return_value = None for child in node.get_children(): if not return_value: t = node.underlying_typedef_type res = t.get_pointee().get_result() type_tokens = self.make_c_style_type_name (res) return_value = [ReturnItemSymbol(type_tokens=type_tokens)] else: type_tokens = self.make_c_style_type_name (child.type) parameter = ParameterSymbol (argname=child.displayname, type_tokens=type_tokens) parameters.append (parameter) if not return_value: return_value = [ReturnItemSymbol(type_tokens=[])] sym = self.__doc_db.get_or_create_symbol(CallbackSymbol, parameters=parameters, return_value=return_value, display_name=node.spelling, filename=str(node.location.file), lineno=node.location.line) return sym def __parse_public_fields (self, decl): tokens = decl.translation_unit.get_tokens(extent=decl.extent) delimiters = [] filename = str(decl.location.file) start = decl.extent.start.line end = decl.extent.end.line + 1 original_lines = [linecache.getline(filename, i).rstrip() for i in range(start, end)] public = True if (self.__locate_delimiters(tokens, delimiters)): public = False children = [] for child in decl.get_children(): children.append(child) delimiters.reverse() if not delimiters: return '\n'.join (original_lines), children public_children = [] children = [] for child in decl.get_children(): children.append(child) children.reverse() if children: next_child = children.pop() else: next_child = None next_delimiter = delimiters.pop() final_text = [] found_first_child = False for i, line in enumerate(original_lines): lineno = i + start if next_delimiter and lineno == next_delimiter[1]: public = next_delimiter[0] if delimiters: next_delimiter = delimiters.pop() else: next_delimiter = None continue if not next_child or lineno < next_child.location.line: if public or not found_first_child: final_text.append (line) continue if lineno == next_child.location.line: found_first_child = True if public: final_text.append (line) public_children.append (next_child) while next_child.location.line == lineno: if not children: public = True next_child = None break next_child = children.pop() return ('\n'.join(final_text), public_children) def __locate_delimiters (self, tokens, delimiters): public_pattern = "/*<public>*/" private_pattern = "/*<private>*/" protected_pattern = "/*<protected>*/" had_public = False for tok in tokens: if tok.kind == clang.cindex.TokenKind.COMMENT: comment = ''.join(tok.spelling.split()) if public_pattern == comment: had_public = True delimiters.append((True, tok.location.line)) elif private_pattern == comment: delimiters.append((False, tok.location.line)) elif protected_pattern == comment: delimiters.append((False, tok.location.line)) return had_public def __create_struct_symbol (self, node, spelling=None): spelling = spelling or node.spelling raw_text, public_fields = self.__parse_public_fields (node) raw_text = unicode(raw_text.decode('utf8')) members = [] for field in public_fields: type_tokens = self.make_c_style_type_name (field.type) is_function_pointer = ast_node_is_function_pointer (field.type) qtype = QualifiedSymbol(type_tokens=type_tokens) name = '%s.%s' % (spelling, field.spelling) member = FieldSymbol (is_function_pointer=is_function_pointer, member_name=field.spelling, qtype=qtype, display_name=name, unique_name=name) members.append (member) if not public_fields: raw_text = None anonymous = not node.spelling return self.__doc_db.get_or_create_symbol(StructSymbol, raw_text=raw_text, members=members, anonymous=anonymous, display_name=spelling, filename=str(node.location.file), lineno=node.location.line) def __create_enum_symbol (self, node, spelling=None): spelling = spelling or node.spelling members = [] for member in node.get_children(): member_value = member.enum_value # FIXME: this is pretty much a macro symbol ? member = self.__doc_db.get_or_create_symbol(Symbol, display_name=member.spelling, filename=str(member.location.file), lineno=member.location.line) member.enum_value = member_value members.append (member) anonymous = not node.spelling start = node.extent.start.line end = node.extent.end.line + 1 original_lines = [linecache.getline(str(node.location.file), i).rstrip() for i in range(start, end)] raw_text = '\n'.join(original_lines) return self.__doc_db.get_or_create_symbol(EnumSymbol, members=members, anonymous=anonymous, raw_text=raw_text, display_name=spelling, filename=str(node.location.file), lineno=node.location.line) def __create_alias_symbol (self, node): type_tokens = self.make_c_style_type_name(node.underlying_typedef_type) aliased_type = QualifiedSymbol (type_tokens=type_tokens) return self.__doc_db.get_or_create_symbol(AliasSymbol, aliased_type=aliased_type, display_name=node.spelling, filename=str(node.location.file), lineno=node.location.line) def __create_typedef_symbol (self, node): t = node.underlying_typedef_type decl = t.get_declaration() if ast_node_is_function_pointer (t): sym = self.__create_callback_symbol (node) elif not decl.spelling and decl.kind == clang.cindex.CursorKind.STRUCT_DECL: # typedef struct {} foo; sym = self.__create_struct_symbol (decl, spelling=node.spelling) elif not decl.spelling and decl.kind == clang.cindex.CursorKind.ENUM_DECL: # typedef enum {} bar; sym = self.__create_enum_symbol (decl, spelling=node.spelling) else: sym = self.__create_alias_symbol (node) return sym def __create_macro_from_raw_text(self, raw): mcontent = raw[0].replace('\t', ' ') mcontent = mcontent.split(' ', 1)[1] split = mcontent.split('(', 1) name = split[0] if not (' ' in name or '\t' in name) and len(split) == 2: args = split[1].split(')', 1)[0].split(',') if args: stripped_name = name.strip() return self.__create_function_macro_symbol(stripped_name, raw[1], raw[2], raw[0]) name = mcontent.split(' ', 1)[0] stripped_name = name.strip() return self.__create_constant_symbol(stripped_name, raw[1], raw[2], raw[0]) def __create_function_macro_symbol (self, name, filename, lineno, original_text): comment = self.doc_repo.doc_database.get_comment(name) return_value = [None] if comment: return_tag = comment.tags.get ('returns') if return_tag: return_value = [ReturnItemSymbol ()] parameters = [] if comment: for param_name in comment.params: parameter = ParameterSymbol (argname=param_name) parameters.append (parameter) sym = self.__doc_db.get_or_create_symbol(FunctionMacroSymbol, return_value=return_value, parameters=parameters, original_text=original_text, display_name=name, filename=filename, lineno=lineno) return sym def __create_constant_symbol (self, name, filename, lineno, original_text): return self.__doc_db.get_or_create_symbol(ConstantSymbol, original_text=original_text, display_name=name, filename=filename, lineno=lineno) def __create_function_symbol (self, node): parameters = [] type_tokens = self.make_c_style_type_name (node.result_type) return_value = [ReturnItemSymbol (type_tokens=type_tokens)] for param in node.get_arguments(): type_tokens = self.make_c_style_type_name (param.type) parameter = ParameterSymbol (argname=param.displayname, type_tokens=type_tokens) parameters.append (parameter) sym = self.__doc_db.get_or_create_symbol(FunctionSymbol, parameters=parameters, return_value=return_value, display_name=node.spelling, filename=str(node.location.file), lineno=node.location.line, extent_start=node.extent.start.line, extent_end=node.extent.end.line) return sym def __create_exported_variable_symbol (self, node): l = linecache.getline (str(node.location.file), node.location.line) split = l.split() start = node.extent.start.line end = node.extent.end.line + 1 filename = str(node.location.file) original_lines = [linecache.getline(filename, i).rstrip() for i in range(start, end)] original_text = '\n'.join(original_lines) type_tokens = self.make_c_style_type_name(node.type) type_qs = QualifiedSymbol(type_tokens=type_tokens) sym = self.__doc_db.get_or_create_symbol(ExportedVariableSymbol, original_text=original_text, display_name=node.spelling, filename=str(node.location.file), lineno=node.location.line, type_qs=type_qs) return sym
class DBusScanner(object): def __init__(self, doc_repo, ext, sources): self.__current_filename = None self.symbols = {} self.doc_repo = doc_repo self.__ext = ext self.__raw_comment_parser = GtkDocParser(self.doc_repo) for filename in sources: self.__current_filename = filename ip = InterfaceParser(filename) for name, interface in ip.parse().iteritems(): self.__create_class_symbol (interface) for mname, method in interface.methods.iteritems(): self.__create_function_symbol (method) for pname, prop in interface.properties.iteritems(): self.__create_property_symbol (prop) for sname, signal in interface.signals.iteritems(): self.__create_signal_symbol (signal) def __create_parameters (self, nodes, omit_direction=False): parameters = [] for param in nodes: type_tokens = [] if not omit_direction: type_tokens.append (param.direction.upper() + ' ') type_tokens.append (param.type) parameters.append (ParameterSymbol (argname=param.name, type_tokens=type_tokens)) return parameters def __comment_from_node(self, node, unique_name=None): if node.comment is None: return None lineno = -1 lines = node.comment.split('\n') stripped_lines = [] column_offset = 0 line_offset = 0 for l in lines: nl = l.strip() if not nl and not stripped_lines: line_offset += 1 continue if not column_offset and nl: column_offset = len(l) - len(nl) stripped_lines.append(nl) if hasattr(node, 'comment_lineno'): lineno = node.comment_lineno + line_offset comment = u'\n'.join(stripped_lines) comment = self.__raw_comment_parser.parse_comment (comment, self.__current_filename, lineno, -1, stripped=True) if comment: comment.col_offset = column_offset + 1 for param in comment.params.values(): param.col_offset = comment.col_offset if unique_name and comment: comment.name = unique_name return comment def __create_function_symbol (self, node): unique_name = '%s.%s' % (self.__current_class_name, node.name) comment = self.__comment_from_node(node, unique_name) self.doc_repo.doc_database.add_comment(comment) parameters = self.__create_parameters (node.arguments) self.__ext.get_or_create_symbol(FunctionSymbol, parameters=parameters, display_name=node.name, filename=self.__current_filename, unique_name=unique_name) def __create_class_symbol (self, node): self.__current_class_name = node.name comment = self.__comment_from_node(node, node.name) self.doc_repo.doc_database.add_comment(comment) self.__ext.get_or_create_symbol(ClassSymbol, display_name=node.name, filename=self.__current_filename) def __create_property_symbol (self, node): unique_name = '%s.%s' % (self.__current_class_name, node.name) comment = self.__comment_from_node(node, unique_name) self.doc_repo.doc_database.add_comment(comment) type_tokens = [node.type] type_ = QualifiedSymbol (type_tokens=type_tokens) flags = '' if node.access == node.ACCESS_READ: flags = 'Read' elif node.access == node.ACCESS_WRITE: flags = 'Write' elif node.access == node.ACCESS_READWRITE: flags = 'Read / Write' sym = self.__ext.get_or_create_symbol(PropertySymbol, prop_type=type_, display_name=node.name, unique_name=unique_name, filename=self.__current_filename) if flags: sym.extension_contents['Flags'] = flags def __create_signal_symbol (self, node): unique_name = '%s.%s' % (self.__current_class_name, node.name) comment = self.__comment_from_node(node, unique_name) self.doc_repo.doc_database.add_comment(comment) parameters = self.__create_parameters (node.arguments, omit_direction=True) self.__ext.get_or_create_symbol(SignalSymbol, parameters=parameters, display_name=node.name, unique_name=unique_name, filename=self.__current_filename)