def test_class_implementations_error(self): # LibClang will fail in parse this source file (it's modified from the original # test/server/mocks.cc from Envoy repository) if we don't add flag PARSE_SKIP_FUNCTION_BODIES # to ignore function bodies. impl_translation_unit = TranslationUnit.from_source( "tools/envoy_headersplit/code_corpus/fail_mocks.cc") impls_cursors = headersplit.class_implementations(impl_translation_unit.cursor) # impls_name is not complete in this case impls_names = [cursor.spelling for cursor in impls_cursors] # LibClang will stop parsing at # MockListenerComponentFactory::MockListenerComponentFactory() # : socket_(std::make_shared<NiceMock<Network::MockListenSocket>>()) { # ^ # Since parsing stops early, we will have incomplete method list. # The reason is not clear, however, this issue can be addressed by adding parsing flag to # ignore function body # get correct list of member methods impl_translation_unit_correct = TranslationUnit.from_source( "tools/envoy_headersplit/code_corpus/fail_mocks.cc", options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) impls_cursors_correct = headersplit.class_implementations( impl_translation_unit_correct.cursor) impls_names_correct = [cursor.spelling for cursor in impls_cursors_correct] self.assertNotEqual(impls_names, impls_names_correct)
def main(args): """ divides the monolithic mock file into different mock class files. """ decl_filename = args["decl"] impl_filename = args["impl"] idx = Index.create() impl_translation_unit = TranslationUnit.from_source( impl_filename, options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) impl_includes = get_directives(impl_translation_unit) decl_translation_unit = idx.parse(decl_filename, ["-x", "c++"]) defns = class_definitions(decl_translation_unit.cursor) decl_includes = get_directives(decl_translation_unit) impl_cursors = class_implementations(impl_translation_unit.cursor) contents = read_file_contents(impl_filename) classname_to_impl = extract_implementations(impl_cursors, contents) classnames = [cursor.spelling for cursor in defns] for defn in defns: # writing {class}.h and {classname}.cc class_name, class_defn, deps = extract_definition(defn, classnames) includes = "" for name in deps: includes += '#include "{}.h"\n'.format(to_filename(name)) class_defn = decl_includes + includes + class_defn class_impl = "" if class_name not in classname_to_impl: print("Warning: empty class {}".format(class_name)) else: impl_include = impl_includes.replace( decl_filename, "{}.h".format(to_filename(class_name))) # we need to enclose methods with namespaces namespace_prefix, namespace_suffix = get_enclosing_namespace(defn) class_impl = impl_include + namespace_prefix + \ classname_to_impl[class_name] + namespace_suffix write_file_contents(class_name, class_defn, class_impl)
def test_parse_arguments(): path = os.path.join(kInputsDir, 'parse_arguments.c') index = Index.create() tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi'], index=index) spellings = [c.spelling for c in tu.cursor.get_children()] assert spellings[-2] == 'hello' assert spellings[-1] == 'hi'
def test_code_complete(): files = [('fake.c', """ /// Aaa. int test1; /// Bbb. void test2(void); void f() { } """)] tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files, options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION) cr = tu.codeComplete('fake.c', 9, 1, unsaved_files=files, include_brief_comments=True) assert cr is not None assert len(cr.diagnostics) == 0 completions = [] for c in cr.results: completions.append(str(c)) expected = [ "{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.", "{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.", "{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None" ] for c in expected: assert c in completions
def test_comment(): files = [('fake.c', """ /// Aaa. int test1; /// Bbb. /// x void test2(void); void f() { } """)] # make a comment-aware TU tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files, options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION) test1 = get_cursor(tu, 'test1') assert test1 is not None, "Could not find test1." assert test1.type.is_pod() raw = test1.raw_comment brief = test1.brief_comment assert raw == """/// Aaa.""" assert brief == """Aaa.""" test2 = get_cursor(tu, 'test2') raw = test2.raw_comment brief = test2.brief_comment assert raw == """/// Bbb.\n/// x""" assert brief == """Bbb. x""" f = get_cursor(tu, 'f') raw = f.raw_comment brief = f.brief_comment assert raw is None assert brief is None
def test_fail_from_source(): path = os.path.join(kInputsDir, 'non-existent.cpp') try: tu = TranslationUnit.from_source(path) except TranslationUnitLoadError: tu = None assert tu == None
def get_tu(source, lang="c", all_warnings=False, flags=[]): """Obtain a translation unit from source and language. By default, the translation unit is created from source file "t.<ext>" where <ext> is the default file extension for the specified language. By default it is C, so "t.c" is the default file name. Supported languages are {c, cpp, objc}. all_warnings is a convenience argument to enable all compiler warnings. """ args = list(flags) name = "t.c" if lang == "cpp": name = "t.cpp" args.append("-std=c++11") elif lang == "objc": name = "t.m" elif lang != "c": raise Exception("Unknown language: %s" % lang) if all_warnings: args += ["-Wall", "-Wextra"] return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
def test_reparse_arguments(self): path = os.path.join(kInputsDir, 'parse_arguments.c') tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi']) tu.reparse() spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-2], 'hello') self.assertEqual(spellings[-1], 'hi')
def get_tu(source, lang='c', all_warnings=False, flags=[]): """Obtain a translation unit from source and language. By default, the translation unit is created from source file "t.<ext>" where <ext> is the default file extension for the specified language. By default it is C, so "t.c" is the default file name. Supported languages are {c, cpp, objc}. all_warnings is a convenience argument to enable all compiler warnings. """ args = list(flags) name = 't.c' if lang == 'cpp': name = 't.cpp' args.append('-std=c++11') elif lang == 'objc': name = 't.m' elif lang != 'c': raise Exception('Unknown language: %s' % lang) if all_warnings: args += ['-Wall', '-Wextra'] return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
def test_comment(self): files = [('fake.c', """ /// Aaa. int test1; /// Bbb. /// x void test2(void); void f() { } """)] # make a comment-aware TU tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files, options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION) test1 = get_cursor(tu, 'test1') self.assertIsNotNone(test1, "Could not find test1.") self.assertTrue(test1.type.is_pod()) raw = test1.raw_comment brief = test1.brief_comment self.assertEqual(raw, """/// Aaa.""") self.assertEqual(brief, """Aaa.""") test2 = get_cursor(tu, 'test2') raw = test2.raw_comment brief = test2.brief_comment self.assertEqual(raw, """/// Bbb.\n/// x""") self.assertEqual(brief, """Bbb. x""") f = get_cursor(tu, 'f') raw = f.raw_comment brief = f.brief_comment self.assertIsNone(raw) self.assertIsNone(brief)
def test_code_complete(): index = Index.create() files = [('fake.c', """ /// Aaa. int test1; /// Bbb. void test2(void); void f() { } """)] tu = TranslationUnit.from_source( 'fake.c', ['-std=c99'], unsaved_files=files, options=TranslationUnit. PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION, index=index) cr = tu.codeComplete('fake.c', 9, 1, unsaved_files=files, include_brief_comments=True) expected = [ "{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.", "{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.", "{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None" ] check_completion_results(cr, expected)
def test_unsaved_files_2(): import io index = Index.create() tu = TranslationUnit.from_source('fake.c', unsaved_files = [ ('fake.c', io.StringIO('int x;'))], index=index) spellings = [c.spelling for c in tu.cursor.get_children()] assert spellings[-1] == 'x'
def test_fail_from_source(self): path = os.path.join(kInputsDir, 'non-existent.cpp') try: tu = TranslationUnit.from_source(path) except TranslationUnitLoadError: tu = None self.assertEqual(tu, None)
def test_fail_from_source(): path = os.path.join(kInputsDir, 'non-existent.cpp') try: tu = TranslationUnit.from_source(path) except TranslationUnitLoadError: tu = None assert tu is None
def test_class_implementations(self): translation_unit_class_impl = TranslationUnit.from_source( "tools/envoy_headersplit/code_corpus/class_impl.cc", options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) impls_cursors = headersplit.class_implementations(translation_unit_class_impl.cursor) impls_names = [cursor.spelling for cursor in impls_cursors] self.assertEqual(impls_names, ["getFoo", "val", "DeadBeaf"])
def collect_nodes(basename, constants): functions = {} types = OrderedDict() # order is important, otherwise the builders will refer to non-existing types. clang_args = [arg.encode('utf-8') for arg in constants.PKG_CONFIG_RES] include_paths = [arg[2:] for arg in constants.PKG_CONFIG_RES if arg[:2] == '-I'] tu_name = join(C_FILES, basename + C_EXT) tu = TranslationUnit.from_source(tu_name, args=clang_args) for node in tu.cursor.get_children(): name = name_of(node) if any(regex.match(name) for regex in constants.BLACKLISTED): continue source_file = node.location.file if source_file: source_filename = source_file.name.decode('utf-8') if not any(source_filename.startswith(whitelist) for whitelist in constants.HEADER_WHITELIST): continue kind = node.kind if kind == CursorKind.FUNCTION_DECL: functions[name] = node elif kind in {CursorKind.STRUCT_DECL, CursorKind.ENUM_DECL}: types[name] = node return (types.values(), functions.values())
def test_code_complete(): index = Index.create() files = [ ( "fake.c", """ /// Aaa. int test1; /// Bbb. void test2(void); void f() { } """, ) ] tu = TranslationUnit.from_source( "fake.c", ["-std=c99"], unsaved_files=files, options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION, index=index, ) cr = tu.codeComplete("fake.c", 9, 1, unsaved_files=files, include_brief_comments=True) expected = [ "{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.", "{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.", "{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None", ] check_completion_results(cr, expected)
def test_parse_arguments(self): path = os.path.join(kInputsDir, 'parse_arguments.c') tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi']) spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-2], 'hello') self.assertEqual(spellings[-1], 'hi')
def find_candidates(filename, ast): """ Find patterns in 'filename' matching 'ast' """ tu = TranslationUnit.from_source(filename, ["-std=c++11"]) for cursor in resolve_ast(tu, ast): yield cursor.location.line
def test_fail_from_source(): path = os.path.join(kInputsDir, 'non-existent.cpp') try: index = Index.create() tu = TranslationUnit.from_source(path, index=index) except TranslationUnitLoadError: tu = None assert tu == None
def test_unsaved_files_2(): import StringIO tu = TranslationUnit.from_source('fake.c', unsaved_files=[ ('fake.c', StringIO.StringIO('int x;')) ]) spellings = [c.spelling for c in tu.cursor.get_children()] assert spellings[-1] == 'x'
def test_unsaved_files_2(): try: from StringIO import StringIO except: from io import StringIO tu = TranslationUnit.from_source('fake.c', unsaved_files = [ ('fake.c', StringIO('int x;'))]) spellings = [c.spelling for c in tu.cursor.get_children()] assert spellings[-1] == 'x'
def test_unsaved_files_2(self): try: from StringIO import StringIO except: from io import StringIO tu = TranslationUnit.from_source('fake.c', unsaved_files = [ ('fake.c', StringIO('int x;'))]) spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-1], 'x')
def parse_file(file): # transform our file into an array of lines with open(file) as f: lines = [line for line in f] # creating our translation unit tu = TranslationUnit.from_source(file) return parse_node(tu.cursor, lines)
def test_unsaved_files_2(self): if sys.version_info.major >= 3: from io import StringIO else: from io import BytesIO as StringIO tu = TranslationUnit.from_source('fake.c', unsaved_files=[('fake.c', StringIO('int x;'))]) spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-1], 'x')
def test_inclusion_directive(self): src = os.path.join(kInputsDir, 'include.cpp') h1 = os.path.join(kInputsDir, "header1.h") h2 = os.path.join(kInputsDir, "header2.h") h3 = os.path.join(kInputsDir, "header3.h") inc = [h1, h3, h2, h3, h1] tu = TranslationUnit.from_source(src, options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD) inclusion_directive_files = [c.get_included_file().name for c in tu.cursor.get_children() if c.kind == CursorKind.INCLUSION_DIRECTIVE] for i in zip(inc, inclusion_directive_files): self.assert_normpaths_equal(i[0], i[1])
def test_unsaved_files(self): tu = TranslationUnit.from_source('fake.c', ['-I./'], unsaved_files=[('fake.c', """ #include "fake.h" int x; int SOME_DEFINE; """), ('./fake.h', """ #define SOME_DEFINE y """)]) spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-2], 'x') self.assertEqual(spellings[-1], 'y')
def test_from_source_accepts_pathlike(self): tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-Iincludes'], unsaved_files=[(str_to_path('fake.c'), """ #include "fake.h" int x; int SOME_DEFINE; """), (str_to_path('includes/fake.h'), """ #define SOME_DEFINE y """)]) spellings = [c.spelling for c in tu.cursor.get_children()] self.assertEqual(spellings[-2], 'x') self.assertEqual(spellings[-1], 'y')
def from_source(cls, source: Text, extra_flags: Optional[List[Text]] = None): flags = ['-x', 'c++', '-std=c++17'] if extra_flags: flags += extra_flags tu = TranslationUnit.from_source('clif_referenced_headers.h', flags, unsaved_files=[ ('clif_referenced_headers.h', source) ]) return cls(tu)
def _collect_nodes(self): types = [] functions = [] tu = TranslationUnit.from_source(join(C_FILES, self._module + C_EXT)) for node in tu.cursor.get_children(): kind = node.kind if kind == CursorKind.FUNCTION_DECL: if not is_blacklisted(name_of(node)): functions.append(node) elif kind in {CursorKind.STRUCT_DECL, CursorKind.ENUM_DECL}: types.append(node) return (types, functions)
def test_get_directives(self): includes = """// your first c++ program // NOLINT(namespace-envoy) #include <iostream> // random strings #include "foo/bar" """ translation_unit_hello_world = TranslationUnit.from_source( "tools/envoy_headersplit/code_corpus/hello.h", options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) self.assertEqual(headersplit.get_directives(translation_unit_hello_world), includes)
def emit_swift(self): cmakeArgs = ["-ObjC"] macOsVersion = float('.'.join( platform.mac_ver()[0].split('.')[:2])) # poor man's version fetch if macOsVersion >= 10.13: cmakeArgs.append( "-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/" ) cmakeArgs.append( "-F/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/" ) translation_unit = TranslationUnit.from_source(self.file_path, args=cmakeArgs) self.swift_file = open('{}.swift'.format(self.app_name), 'w') for inclusion in translation_unit.get_includes(): if inclusion.depth == 1: include = inclusion.include.name self.emit_line('import {}'.format(name_from_path(include))) self.emit_line(base_protocols) cursor = translation_unit.cursor local_children = [ child for child in cursor.get_children() if child.location.file and child.location.file.name == self.file_path ] enums = [ child for child in local_children if child.kind == CursorKind.ENUM_DECL ] self.emit_enums(enums) for child in [ child for child in local_children if child.kind == CursorKind.OBJC_PROTOCOL_DECL ]: self.emit_protocol(child) categories = [ child for child in local_children if child.kind == CursorKind.OBJC_CATEGORY_DECL ] self.gather_categories(categories) for child in [ child for child in local_children if child.kind == CursorKind.OBJC_INTERFACE_DECL ]: self.emit_protocol(child) self.swift_file.close()
def test_unsaved_files(): index = Index.create() tu = TranslationUnit.from_source('fake.c', ['-I./'], unsaved_files = [ ('fake.c', """ #include "fake.h" int x; int SOME_DEFINE; """), ('./fake.h', """ #define SOME_DEFINE y """) ], index=index) spellings = [c.spelling for c in tu.cursor.get_children()] assert spellings[-2] == 'x' assert spellings[-1] == 'y'
def get_named_tu(source, name, all_warnings=False, flags=[]): """Obtain a translation unit from source and filename. Language is deduced from the filename. The filename does not need to correspond to a real file but will be the name of an unsaved translation unit. """ args = list(flags) if name.endswith('cpp') or name.endswith('.cxx'): args.extend('-x c++ -std=c++11 -stdlib=libc++'.split()) if all_warnings: args += ['-Wall', '-Wextra'] return TranslationUnit.from_source( name, args, unsaved_files=[(name, source)])
def test_includes(self): def eq(expected, actual): if not actual.is_input_file: self.assert_normpaths_equal(expected[0], actual.source.name) self.assert_normpaths_equal(expected[1], actual.include.name) else: self.assert_normpaths_equal(expected[1], actual.include.name) src = os.path.join(kInputsDir, 'include.cpp') h1 = os.path.join(kInputsDir, "header1.h") h2 = os.path.join(kInputsDir, "header2.h") h3 = os.path.join(kInputsDir, "header3.h") inc = [(src, h1), (h1, h3), (src, h2), (h2, h3)] tu = TranslationUnit.from_source(src) for i in zip(inc, tu.get_includes()): eq(i[0], i[1])
def from_source(cls, source: Text, extra_flags: Optional[List[Text]] = None): """Create a Module object from C++ source file.""" flags = ['-x', 'c++', '-std=c++17', '-I.'] if extra_flags: flags += extra_flags tu = TranslationUnit.from_source('clif_referenced_headers.h', flags, unsaved_files=[ ('clif_referenced_headers.h', source) ]) if tu.diagnostics: err_msg = '\n'.join(str(e) for e in tu.diagnostics) raise ValueError(f'Errors in source file: {err_msg}') return cls(tu)
def test_includes(): def eq(expected, actual): if not actual.is_input_file: return normpaths_equal(expected[0], actual.source.name) and \ normpaths_equal(expected[1], actual.include.name) else: return normpaths_equal(expected[1], actual.include.name) src = os.path.join(kInputsDir, 'include.cpp') h1 = os.path.join(kInputsDir, "header1.h") h2 = os.path.join(kInputsDir, "header2.h") h3 = os.path.join(kInputsDir, "header3.h") inc = [(src, h1), (h1, h3), (src, h2), (h2, h3)] index = Index.create() tu = TranslationUnit.from_source(src, index=index) for i in zip(inc, tu.get_includes()): assert eq(i[0], i[1])
def try_clang_cindex(): from clang.cindex import Index, Config, TranslationUnit Config.set_library_file(f"/usr/lib/llvm-9/lib/libclang.so") filename = "/tmp/clang_cindex_tmp_src.cc" with open(filename, "w") as f: f.write(code) index = Index.create() t_start = time.time() tu = TranslationUnit.from_source( filename=filename, index=index, args=[ f"-I/usr/include/eigen3", ], ) dt = time.time() - t_start return dt
def test_code_complete_availability(): index = Index.create() files = [('fake.cpp', """ class P { protected: int member; }; class Q : public P { public: using P::member; }; void f(P x, Q y) { x.; // member is inaccessible y.; // member is accessible } """)] tu = TranslationUnit.from_source('fake.cpp', ['-std=c++98'], unsaved_files=files, index=index) cr = tu.codeComplete('fake.cpp', 12, 5, unsaved_files=files) expected = [ "{'const', TypedText} || Priority: 40 || Availability: Available || Brief comment: None", "{'volatile', TypedText} || Priority: 40 || Availability: Available || Brief comment: None", "{'operator', TypedText} || Priority: 40 || Availability: Available || Brief comment: None", "{'P', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None", "{'Q', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None" ] check_completion_results(cr, expected) cr = tu.codeComplete('fake.cpp', 13, 5, unsaved_files=files) expected = [ "{'P', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None", "{'P &', ResultType} | {'operator=', TypedText} | {'(', LeftParen} | {'const P &', Placeholder} | {')', RightParen} || Priority: 34 || Availability: Available || Brief comment: None", "{'int', ResultType} | {'member', TypedText} || Priority: 35 || Availability: NotAccessible || Brief comment: None", "{'void', ResultType} | {'~P', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 34 || Availability: Available || Brief comment: None" ] check_completion_results(cr, expected)
def test_code_complete(): files = [('fake.c', """ /// Aaa. int test1; /// Bbb. void test2(void); void f() { } """)] tu = TranslationUnit.from_source( 'fake.c', ['-std=c99'], unsaved_files=files, options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION ) cr = tu.codeComplete('fake.c', 9, 1, unsaved_files=files, include_brief_comments=True) assert cr is not None assert len(cr.diagnostics) == 0 completions = [] for c in cr.results: completions.append(str(c)) expected = [ "{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.", "{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.", "{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None" ] for c in expected: assert c in completions