def test_class_implementations_error(self):
# LibClang will fail in parse this source file (it's modified from the original
# test/server/mocks.cc from Envoy repository) if we don't add flag PARSE_SKIP_FUNCTION_BODIES
# to ignore function bodies.
impl_translation_unit = TranslationUnit.from_source(
"tools/envoy_headersplit/code_corpus/fail_mocks.cc")
impls_cursors = headersplit.class_implementations(impl_translation_unit.cursor)
# impls_name is not complete in this case
impls_names = [cursor.spelling for cursor in impls_cursors]
# LibClang will stop parsing at
# MockListenerComponentFactory::MockListenerComponentFactory()
# : socket_(std::make_shared<NiceMock<Network::MockListenSocket>>()) {
# ^
# Since parsing stops early, we will have incomplete method list.
# The reason is not clear, however, this issue can be addressed by adding parsing flag to
# ignore function body
# get correct list of member methods
impl_translation_unit_correct = TranslationUnit.from_source(
"tools/envoy_headersplit/code_corpus/fail_mocks.cc",
options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
impls_cursors_correct = headersplit.class_implementations(
impl_translation_unit_correct.cursor)
impls_names_correct = [cursor.spelling for cursor in impls_cursors_correct]
self.assertNotEqual(impls_names, impls_names_correct)
def main():
# type: () -> None
parser = argparse.ArgumentParser()
parser.add_argument("--genpybind-module", dest="module", required=True)
parser.add_argument("--genpybind-parse", dest="executable", default="genpybind-parse")
parser.add_argument("--genpybind-docstring", dest="docstring")
parser.add_argument("--genpybind-include", nargs="+", dest="includes")
parser.add_argument("--genpybind-isystem", nargs="+", dest="isystem")
parser.add_argument("--genpybind-tag", nargs="+", dest="tags")
parser.add_argument("--genpybind-from-ast", dest="from_ast")
parser.add_argument('rest', nargs=argparse.REMAINDER)
# args, rest_args = parser.parse_known_args()
args = parser.parse_args()
if args.from_ast:
if args.rest:
parser.error(
"unexpected arguments with --genpybind-from-ast: {}".format(args.rest))
translation_unit = TranslationUnit.from_ast_file(args.from_ast)
else:
name = tempfile.mkdtemp(prefix="genpybind")
ast_file = os.path.join(name, "genpybind.ast")
try:
rest = args.rest[:]
if rest[0] == "--":
del rest[0]
status = subprocess.call(
[args.executable, "-output-file", ast_file] + rest, stdout=sys.stderr)
if status != 0:
parser.error("genpybind-parse returned status {} when called with\n{}".format(
status, rest))
translation_unit = TranslationUnit.from_ast_file(ast_file)
finally:
shutil.rmtree(name)
if translation_unit.diagnostics:
for diag in translation_unit.diagnostics:
print("//", diag.format())
for diag_ in diag.children:
print("//", " ", diag_.format())
toplevel_declarations = gather_declarations(translation_unit.cursor)
print(expose_as(
toplevel_declarations,
module=args.module,
doc=args.docstring,
isystem=args.isystem,
includes=args.includes,
tags=args.tags,
))
def get_tu(source, lang='c', all_warnings=False, flags=[]):
"""Obtain a translation unit from source and language.
By default, the translation unit is created from source file "t.<ext>"
where <ext> is the default file extension for the specified language. By
default it is C, so "t.c" is the default file name.
Supported languages are {c, cpp, objc}.
all_warnings is a convenience argument to enable all compiler warnings.
"""
args = list(flags)
name = 't.c'
if lang == 'cpp':
name = 't.cpp'
args.append('-std=c++11')
elif lang == 'objc':
name = 't.m'
elif lang != 'c':
raise Exception('Unknown language: %s' % lang)
if all_warnings:
args += ['-Wall', '-Wextra']
return TranslationUnit.from_source(name,
args,
unsaved_files=[(name, source)])
def test_fail_from_source(self):
path = os.path.join(kInputsDir, 'non-existent.cpp')
try:
tu = TranslationUnit.from_source(path)
except TranslationUnitLoadError:
tu = None
self.assertEqual(tu, None)
def test_fail_from_ast_file():
path = os.path.join(kInputsDir, 'non-existent.ast')
try:
tu = TranslationUnit.from_ast_file(path)
except TranslationUnitLoadError:
tu = None
assert tu == None
def test_code_complete():
files = [('fake.c', """
/// Aaa.
int test1;
/// Bbb.
void test2(void);
void f() {
}
""")]
tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
cr = tu.codeComplete('fake.c', 9, 1, unsaved_files=files, include_brief_comments=True)
assert cr is not None
assert len(cr.diagnostics) == 0
completions = []
for c in cr.results:
completions.append(str(c))
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None"
]
for c in expected:
assert c in completions
def test_class_implementations(self):
translation_unit_class_impl = TranslationUnit.from_source(
"tools/envoy_headersplit/code_corpus/class_impl.cc",
options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
impls_cursors = headersplit.class_implementations(translation_unit_class_impl.cursor)
impls_names = [cursor.spelling for cursor in impls_cursors]
self.assertEqual(impls_names, ["getFoo", "val", "DeadBeaf"])
def parse_files(list_of_files: List[str]) -> Tuple[Any, List[Any]]:
"""
Use Clang to parse the provided list of files.
"""
idx = Index.create()
t_units = [] # type: List[Any]
# To avoid parsing the files all the time, store the parsed ASTs
# of each compilation unit in a cache.
if not os.path.exists(".cache"):
os.mkdir(".cache")
for i, filename in enumerate(list_of_files):
cache_filename = '.cache/' + os.path.basename(filename) + "_cache"
# Have I parsed this file before?
if os.path.exists(cache_filename):
# Yes, load from cache
try:
t_units.append(
TranslationUnit.from_ast_file(cache_filename, idx))
logging.info("[-] Loading cached AST for %s", filename)
except TranslationUnitLoadError: # pragma: nocover
logging.info("[-] %3d%% Parsing %s",
(100 * (i + 1) / len(list_of_files)),
filename) # pragma: nocover
t_units.append(idx.parse(filename)) # pragma: nocover
t_units[-1].save(cache_filename) # pragma: nocover
else:
# No, parse it now.
logging.info("[-] %3d%% Parsing %s",
(100 * (i + 1) / len(list_of_files)), filename)
# ...to adapt to: .... filename, args=['-I ...', '-D ...']
t_units.append(idx.parse(filename))
t_units[-1].save(cache_filename)
return idx, t_units
def test_code_complete():
index = Index.create()
files = [('fake.c', """
/// Aaa.
int test1;
/// Bbb.
void test2(void);
void f() {
}
""")]
tu = TranslationUnit.from_source(
'fake.c', ['-std=c99'],
unsaved_files=files,
options=TranslationUnit.
PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION,
index=index)
cr = tu.codeComplete('fake.c',
9,
1,
unsaved_files=files,
include_brief_comments=True)
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None"
]
check_completion_results(cr, expected)
def get_tu(source, lang='c', all_warnings=False, flags=[]):
"""Obtain a translation unit from source and language.
By default, the translation unit is created from source file "t.<ext>"
where <ext> is the default file extension for the specified language. By
default it is C, so "t.c" is the default file name.
Supported languages are {c, cpp, objc}.
all_warnings is a convenience argument to enable all compiler warnings.
"""
args = list(flags)
name = 't.c'
if lang == 'cpp':
name = 't.cpp'
args.append('-std=c++11')
elif lang == 'objc':
name = 't.m'
elif lang != 'c':
raise Exception('Unknown language: %s' % lang)
if all_warnings:
args += ['-Wall', '-Wextra']
return TranslationUnit.from_source(name, args, unsaved_files=[(name,
source)])
def parse_files(list_of_files: List[str]) -> Tuple[Any, List[Any]]:
"""
Use Clang to parse the provided list of files, and return
a tuple of the Clang index, and the list of compiled ASTs
(one for each compilation unit)
"""
idx = Index.create()
t_units = [] # type: List[Any]
# To avoid parsing the files all the time, store the parsed ASTs
# of each compilation unit in a cache.
if not os.path.exists(".cache"):
os.mkdir(".cache")
for i, filename in enumerate(list_of_files):
cache_filename = '.cache/' + os.path.basename(filename) + "_cache"
# Have I parsed this file before?
if os.path.exists(cache_filename):
# Yes, load from cache
try:
t_units.append(
TranslationUnit.from_ast_file(
cache_filename, idx))
print("[-] Loading cached AST for", filename)
except TranslationUnitLoadError:
print("[-] %3d%% Parsing " % (
100*(i+1)/len(list_of_files)) + filename)
t_units.append(idx.parse(filename))
t_units[-1].save(cache_filename)
else:
# No, parse it now.
print("[-] %3d%% Parsing " % (
100*(i+1)/len(list_of_files)) + filename)
t_units.append(idx.parse(filename))
t_units[-1].save(cache_filename)
return idx, t_units
def main(args):
"""
divides the monolithic mock file into different mock class files.
"""
decl_filename = args["decl"]
impl_filename = args["impl"]
idx = Index.create()
impl_translation_unit = TranslationUnit.from_source(
impl_filename, options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
impl_includes = get_directives(impl_translation_unit)
decl_translation_unit = idx.parse(decl_filename, ["-x", "c++"])
defns = class_definitions(decl_translation_unit.cursor)
decl_includes = get_directives(decl_translation_unit)
impl_cursors = class_implementations(impl_translation_unit.cursor)
contents = read_file_contents(impl_filename)
classname_to_impl = extract_implementations(impl_cursors, contents)
classnames = [cursor.spelling for cursor in defns]
for defn in defns:
# writing {class}.h and {classname}.cc
class_name, class_defn, deps = extract_definition(defn, classnames)
includes = ""
for name in deps:
includes += '#include "{}.h"\n'.format(to_filename(name))
class_defn = decl_includes + includes + class_defn
class_impl = ""
if class_name not in classname_to_impl:
print("Warning: empty class {}".format(class_name))
else:
impl_include = impl_includes.replace(
decl_filename, "{}.h".format(to_filename(class_name)))
# we need to enclose methods with namespaces
namespace_prefix, namespace_suffix = get_enclosing_namespace(defn)
class_impl = impl_include + namespace_prefix + \
classname_to_impl[class_name] + namespace_suffix
write_file_contents(class_name, class_defn, class_impl)
def collect_nodes(basename, constants):
functions = {}
types = OrderedDict()
# order is important, otherwise the builders will refer to non-existing types.
clang_args = [arg.encode('utf-8') for arg in constants.PKG_CONFIG_RES]
include_paths = [arg[2:] for arg in constants.PKG_CONFIG_RES if arg[:2] == '-I']
tu_name = join(C_FILES, basename + C_EXT)
tu = TranslationUnit.from_source(tu_name, args=clang_args)
for node in tu.cursor.get_children():
name = name_of(node)
if any(regex.match(name) for regex in constants.BLACKLISTED):
continue
source_file = node.location.file
if source_file:
source_filename = source_file.name.decode('utf-8')
if not any(source_filename.startswith(whitelist) for whitelist in constants.HEADER_WHITELIST):
continue
kind = node.kind
if kind == CursorKind.FUNCTION_DECL:
functions[name] = node
elif kind in {CursorKind.STRUCT_DECL, CursorKind.ENUM_DECL}:
types[name] = node
return (types.values(), functions.values())
def test_reparse_arguments(self):
path = os.path.join(kInputsDir, 'parse_arguments.c')
tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi'])
tu.reparse()
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'hello')
self.assertEqual(spellings[-1], 'hi')
def test_unsaved_files_2():
import io
index = Index.create()
tu = TranslationUnit.from_source('fake.c', unsaved_files = [
('fake.c', io.StringIO('int x;'))], index=index)
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-1] == 'x'
def test_code_complete():
index = Index.create()
files = [
(
"fake.c",
"""
/// Aaa.
int test1;
/// Bbb.
void test2(void);
void f() {
}
""",
)
]
tu = TranslationUnit.from_source(
"fake.c",
["-std=c99"],
unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION,
index=index,
)
cr = tu.codeComplete("fake.c", 9, 1, unsaved_files=files, include_brief_comments=True)
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
]
check_completion_results(cr, expected)
def test_fail_from_source():
path = os.path.join(kInputsDir, 'non-existent.cpp')
try:
tu = TranslationUnit.from_source(path)
except TranslationUnitLoadError:
tu = None
assert tu == None
def find_candidates(filename, ast):
""" Find patterns in 'filename' matching 'ast'
"""
tu = TranslationUnit.from_source(filename, ["-std=c++11"])
for cursor in resolve_ast(tu, ast):
yield cursor.location.line
def get_tu(source, lang="c", all_warnings=False, flags=[]):
"""Obtain a translation unit from source and language.
By default, the translation unit is created from source file "t.<ext>"
where <ext> is the default file extension for the specified language. By
default it is C, so "t.c" is the default file name.
Supported languages are {c, cpp, objc}.
all_warnings is a convenience argument to enable all compiler warnings.
"""
args = list(flags)
name = "t.c"
if lang == "cpp":
name = "t.cpp"
args.append("-std=c++11")
elif lang == "objc":
name = "t.m"
elif lang != "c":
raise Exception("Unknown language: %s" % lang)
if all_warnings:
args += ["-Wall", "-Wextra"]
return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
def test_fail_from_ast_file(self):
path = os.path.join(kInputsDir, 'non-existent.ast')
try:
tu = TranslationUnit.from_ast_file(path)
except TranslationUnitLoadError:
tu = None
self.assertEqual(tu, None)
def test_parse_arguments(self):
path = os.path.join(kInputsDir, 'parse_arguments.c')
tu = TranslationUnit.from_source(path,
['-DDECL_ONE=hello', '-DDECL_TWO=hi'])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'hello')
self.assertEqual(spellings[-1], 'hi')
def test_comment(self):
files = [('fake.c', """
/// Aaa.
int test1;
/// Bbb.
/// x
void test2(void);
void f() {
}
""")]
# make a comment-aware TU
tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
test1 = get_cursor(tu, 'test1')
self.assertIsNotNone(test1, "Could not find test1.")
self.assertTrue(test1.type.is_pod())
raw = test1.raw_comment
brief = test1.brief_comment
self.assertEqual(raw, """/// Aaa.""")
self.assertEqual(brief, """Aaa.""")
test2 = get_cursor(tu, 'test2')
raw = test2.raw_comment
brief = test2.brief_comment
self.assertEqual(raw, """/// Bbb.\n/// x""")
self.assertEqual(brief, """Bbb. x""")
f = get_cursor(tu, 'f')
raw = f.raw_comment
brief = f.brief_comment
self.assertIsNone(raw)
self.assertIsNone(brief)
def test_comment():
files = [('fake.c', """
/// Aaa.
int test1;
/// Bbb.
/// x
void test2(void);
void f() {
}
""")]
# make a comment-aware TU
tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
test1 = get_cursor(tu, 'test1')
assert test1 is not None, "Could not find test1."
assert test1.type.is_pod()
raw = test1.raw_comment
brief = test1.brief_comment
assert raw == """/// Aaa."""
assert brief == """Aaa."""
test2 = get_cursor(tu, 'test2')
raw = test2.raw_comment
brief = test2.brief_comment
assert raw == """/// Bbb.\n/// x"""
assert brief == """Bbb. x"""
f = get_cursor(tu, 'f')
raw = f.raw_comment
brief = f.brief_comment
assert raw is None
assert brief is None
def test_parse_arguments():
path = os.path.join(kInputsDir, 'parse_arguments.c')
index = Index.create()
tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi'], index=index)
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-2] == 'hello'
assert spellings[-1] == 'hi'
def test_fail_from_ast_file():
path = os.path.join(kInputsDir, 'non-existent.ast')
try:
index = Index.create()
tu = TranslationUnit.from_ast_file(path, index=index)
except TranslationUnitLoadError:
tu = None
assert tu == None
def test_fail_from_source():
path = os.path.join(kInputsDir, 'non-existent.cpp')
try:
index = Index.create()
tu = TranslationUnit.from_source(path, index=index)
except TranslationUnitLoadError:
tu = None
assert tu == None
def test_unsaved_files_2(self):
if sys.version_info.major >= 3:
from io import StringIO
else:
from io import BytesIO as StringIO
tu = TranslationUnit.from_source('fake.c', unsaved_files = [
('fake.c', StringIO('int x;'))])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-1], 'x')
def test_unsaved_files_2(self):
try:
from StringIO import StringIO
except:
from io import StringIO
tu = TranslationUnit.from_source('fake.c', unsaved_files = [
('fake.c', StringIO('int x;'))])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-1], 'x')
def test_unsaved_files_2():
import StringIO
tu = TranslationUnit.from_source('fake.c',
unsaved_files=[
('fake.c',
StringIO.StringIO('int x;'))
])
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-1] == 'x'
def test_unsaved_files_2():
try:
from StringIO import StringIO
except:
from io import StringIO
tu = TranslationUnit.from_source('fake.c', unsaved_files = [
('fake.c', StringIO('int x;'))])
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-1] == 'x'
def parse_file(file):
# transform our file into an array of lines
with open(file) as f:
lines = [line for line in f]
# creating our translation unit
tu = TranslationUnit.from_source(file)
return parse_node(tu.cursor, lines)
def test_unsaved_files_2(self):
if sys.version_info.major >= 3:
from io import StringIO
else:
from io import BytesIO as StringIO
tu = TranslationUnit.from_source('fake.c',
unsaved_files=[('fake.c',
StringIO('int x;'))])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-1], 'x')
def test_inclusion_directive(self):
src = os.path.join(kInputsDir, 'include.cpp')
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
inc = [h1, h3, h2, h3, h1]
tu = TranslationUnit.from_source(src, options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD)
inclusion_directive_files = [c.get_included_file().name for c in tu.cursor.get_children() if c.kind == CursorKind.INCLUSION_DIRECTIVE]
for i in zip(inc, inclusion_directive_files):
self.assert_normpaths_equal(i[0], i[1])
def test_inclusion_directive(self):
src = os.path.join(kInputsDir, 'include.cpp')
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
inc = [h1, h3, h2, h3, h1]
tu = TranslationUnit.from_source(src, options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD)
inclusion_directive_files = [c.get_included_file().name for c in tu.cursor.get_children() if c.kind == CursorKind.INCLUSION_DIRECTIVE]
for i in zip(inc, inclusion_directive_files):
self.assert_normpaths_equal(i[0], i[1])
def test_unsaved_files(self):
tu = TranslationUnit.from_source('fake.c', ['-I./'],
unsaved_files=[('fake.c', """
#include "fake.h"
int x;
int SOME_DEFINE;
"""), ('./fake.h', """
#define SOME_DEFINE y
""")])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'x')
self.assertEqual(spellings[-1], 'y')
def from_source(cls,
source: Text,
extra_flags: Optional[List[Text]] = None):
flags = ['-x', 'c++', '-std=c++17']
if extra_flags:
flags += extra_flags
tu = TranslationUnit.from_source('clif_referenced_headers.h',
flags,
unsaved_files=[
('clif_referenced_headers.h',
source)
])
return cls(tu)
def test_from_source_accepts_pathlike(self):
tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-Iincludes'],
unsaved_files=[(str_to_path('fake.c'),
"""
#include "fake.h"
int x;
int SOME_DEFINE;
"""), (str_to_path('includes/fake.h'), """
#define SOME_DEFINE y
""")])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'x')
self.assertEqual(spellings[-1], 'y')
def test_unsaved_files():
tu = TranslationUnit.from_source('fake.c', ['-I./'], unsaved_files = [
('fake.c', """
#include "fake.h"
int x;
int SOME_DEFINE;
"""),
('./fake.h', """
#define SOME_DEFINE y
""")
])
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-2] == 'x'
assert spellings[-1] == 'y'
def test_from_source_accepts_pathlike(self):
tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-Iincludes'], unsaved_files = [
(str_to_path('fake.c'), """
#include "fake.h"
int x;
int SOME_DEFINE;
"""),
(str_to_path('includes/fake.h'), """
#define SOME_DEFINE y
""")
])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'x')
self.assertEqual(spellings[-1], 'y')
def test_load_pathlike(self):
"""Ensure TranslationUnits can be constructed from saved files -
PathLike variant."""
tu = get_tu('int foo();')
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
tu2 = TranslationUnit.from_ast_file(filename=str_to_path(path))
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
del tu2
def _collect_nodes(self):
types = []
functions = []
tu = TranslationUnit.from_source(join(C_FILES, self._module + C_EXT))
for node in tu.cursor.get_children():
kind = node.kind
if kind == CursorKind.FUNCTION_DECL:
if not is_blacklisted(name_of(node)):
functions.append(node)
elif kind in {CursorKind.STRUCT_DECL, CursorKind.ENUM_DECL}:
types.append(node)
return (types, functions)
def test_get_directives(self):
includes = """// your first c++ program
// NOLINT(namespace-envoy)
#include <iostream>
// random strings
#include "foo/bar"
"""
translation_unit_hello_world = TranslationUnit.from_source(
"tools/envoy_headersplit/code_corpus/hello.h",
options=TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
self.assertEqual(headersplit.get_directives(translation_unit_hello_world), includes)
def _collect_nodes(self):
types = []
functions = []
tu = TranslationUnit.from_source(join(C_FILES, self._module + C_EXT))
for node in tu.cursor.get_children():
kind = node.kind
if kind == CursorKind.FUNCTION_DECL:
if not is_blacklisted(name_of(node)):
functions.append(node)
elif kind in {CursorKind.STRUCT_DECL, CursorKind.ENUM_DECL}:
types.append(node)
return (types, functions)
def test_load_pathlike(self):
"""Ensure TranslationUnits can be constructed from saved files -
PathLike variant."""
tu = get_tu('int foo();')
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
tu2 = TranslationUnit.from_ast_file(filename=path)
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
del tu2
def emit_swift(self):
cmakeArgs = ["-ObjC"]
macOsVersion = float('.'.join(
platform.mac_ver()[0].split('.')[:2])) # poor man's version fetch
if macOsVersion >= 10.13:
cmakeArgs.append(
"-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/"
)
cmakeArgs.append(
"-F/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/"
)
translation_unit = TranslationUnit.from_source(self.file_path,
args=cmakeArgs)
self.swift_file = open('{}.swift'.format(self.app_name), 'w')
for inclusion in translation_unit.get_includes():
if inclusion.depth == 1:
include = inclusion.include.name
self.emit_line('import {}'.format(name_from_path(include)))
self.emit_line(base_protocols)
cursor = translation_unit.cursor
local_children = [
child for child in cursor.get_children() if child.location.file
and child.location.file.name == self.file_path
]
enums = [
child for child in local_children
if child.kind == CursorKind.ENUM_DECL
]
self.emit_enums(enums)
for child in [
child for child in local_children
if child.kind == CursorKind.OBJC_PROTOCOL_DECL
]:
self.emit_protocol(child)
categories = [
child for child in local_children
if child.kind == CursorKind.OBJC_CATEGORY_DECL
]
self.gather_categories(categories)
for child in [
child for child in local_children
if child.kind == CursorKind.OBJC_INTERFACE_DECL
]:
self.emit_protocol(child)
self.swift_file.close()
def test_unsaved_files():
index = Index.create()
tu = TranslationUnit.from_source('fake.c', ['-I./'], unsaved_files = [
('fake.c', """
#include "fake.h"
int x;
int SOME_DEFINE;
"""),
('./fake.h', """
#define SOME_DEFINE y
""")
], index=index)
spellings = [c.spelling for c in tu.cursor.get_children()]
assert spellings[-2] == 'x'
assert spellings[-1] == 'y'
def _get_completions(self, context, token):
parseopts = TranslationUnit.PARSE_INCOMPLETE
parseopts += TranslationUnit.PARSE_PRECOMPILED_PREAMBLE
parseopts += TranslationUnit.PARSE_CACHE_COMPLETION_RESULTS
parseopts += TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
parseopts += TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION
if not self._can_complete(context):
return []
buf = self._get_buffer(context)
line, column = self._get_pos(buf)
column -= len(token)
cwd = os.getcwd()
self._cd_builddir()
path = self._get_completion_path()
args = self._get_completion_args(context)
src = buf.get_text(buf.get_start_iter(), buf.get_end_iter(), True)
files = [(path, src)]
tu = TranslationUnit.from_source(path, args, unsaved_files=files,
options=parseopts,
index=self.index)
cr = tu.codeComplete(path, line, column, unsaved_files=files,
include_macros=True, include_code_patterns=True,
include_brief_comments=True)
os.chdir(cwd)
completions = []
for result in cr.results:
hint = ''
contents = ''
string = result.string
doc = string.briefComment
for chunk in string:
s = chunk.spelling
if not s:
continue
if not isinstance(s, str):
s = s.decode('utf-8')
if chunk.isKindTypedText():
trigger = s
hint += s
if chunk.isKindResultType():
hint += ' '
else:
contents += s
if len(trigger) and len(hint):
completions.append((trigger, hint, contents, doc))
return completions
def get_named_tu(source, name, all_warnings=False, flags=[]):
"""Obtain a translation unit from source and filename.
Language is deduced from the filename.
The filename does not need to correspond to a real file but will be the
name of an unsaved translation unit.
"""
args = list(flags)
if name.endswith('cpp') or name.endswith('.cxx'):
args.extend('-x c++ -std=c++11 -stdlib=libc++'.split())
if all_warnings:
args += ['-Wall', '-Wextra']
return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
def emit_swift(self):
translation_unit = TranslationUnit.from_source(self.file_path, args=["-ObjC"])
self.swift_file = open('{}.swift'.format(self.app_name), 'w')
for inclusion in translation_unit.get_includes():
if inclusion.depth == 1:
include = inclusion.include.name
self.emit_line('import {}'.format(name_from_path(include)))
self.emit_line(base_protocols)
cursor = translation_unit.cursor
local_children = [child for child in cursor.get_children()
if child.location.file and child.location.file.name == self.file_path]
categories = [child for child in local_children if child.kind == CursorKind.OBJC_CATEGORY_DECL]
self.gather_categories(categories)
for child in [child for child in local_children if child.kind == CursorKind.OBJC_INTERFACE_DECL]:
self.emit_protocol(child)
self.swift_file.close()
def test_includes(self):
def eq(expected, actual):
if not actual.is_input_file:
self.assert_normpaths_equal(expected[0], actual.source.name)
self.assert_normpaths_equal(expected[1], actual.include.name)
else:
self.assert_normpaths_equal(expected[1], actual.include.name)
src = os.path.join(kInputsDir, 'include.cpp')
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
inc = [(src, h1), (h1, h3), (src, h2), (h2, h3)]
tu = TranslationUnit.from_source(src)
for i in zip(inc, tu.get_includes()):
eq(i[0], i[1])
def test_load(self):
"""Ensure TranslationUnits can be constructed from saved files."""
tu = get_tu('int foo();')
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
self.assertTrue(os.path.exists(path))
self.assertGreater(os.path.getsize(path), 0)
tu2 = TranslationUnit.from_ast_file(filename=path)
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
del tu2
def from_source(cls,
source: Text,
extra_flags: Optional[List[Text]] = None):
"""Create a Module object from C++ source file."""
flags = ['-x', 'c++', '-std=c++17', '-I.']
if extra_flags:
flags += extra_flags
tu = TranslationUnit.from_source('clif_referenced_headers.h',
flags,
unsaved_files=[
('clif_referenced_headers.h',
source)
])
if tu.diagnostics:
err_msg = '\n'.join(str(e) for e in tu.diagnostics)
raise ValueError(f'Errors in source file: {err_msg}')
return cls(tu)
def test_includes():
def eq(expected, actual):
if not actual.is_input_file:
return normpaths_equal(expected[0], actual.source.name) and \
normpaths_equal(expected[1], actual.include.name)
else:
return normpaths_equal(expected[1], actual.include.name)
src = os.path.join(kInputsDir, 'include.cpp')
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
inc = [(src, h1), (h1, h3), (src, h2), (h2, h3)]
tu = TranslationUnit.from_source(src)
for i in zip(inc, tu.get_includes()):
assert eq(i[0], i[1])
def test_load(self):
"""Ensure TranslationUnits can be constructed from saved files."""
tu = get_tu('int foo();')
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
self.assertTrue(os.path.exists(path))
self.assertGreater(os.path.getsize(path), 0)
tu2 = TranslationUnit.from_ast_file(filename=path)
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
del tu2
def test_code_complete_availability():
index = Index.create()
files = [
(
"fake.cpp",
"""
class P {
protected:
int member;
};
class Q : public P {
public:
using P::member;
};
void f(P x, Q y) {
x.; // member is inaccessible
y.; // member is accessible
}
""",
)
]
tu = TranslationUnit.from_source("fake.cpp", ["-std=c++98"], unsaved_files=files, index=index)
cr = tu.codeComplete("fake.cpp", 12, 5, unsaved_files=files)
expected = [
"{'const', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
"{'volatile', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
"{'operator', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
"{'P', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None",
"{'Q', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None",
]
check_completion_results(cr, expected)
cr = tu.codeComplete("fake.cpp", 13, 5, unsaved_files=files)
expected = [
"{'P', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None",
"{'P &', ResultType} | {'operator=', TypedText} | {'(', LeftParen} | {'const P &', Placeholder} | {')', RightParen} || Priority: 34 || Availability: Available || Brief comment: None",
"{'int', ResultType} | {'member', TypedText} || Priority: 35 || Availability: NotAccessible || Brief comment: None",
"{'void', ResultType} | {'~P', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 34 || Availability: Available || Brief comment: None",
]
check_completion_results(cr, expected)
def test_load():
"""Ensure TranslationUnits can be constructed from saved files."""
tu = get_tu('int foo();')
assert len(tu.diagnostics) == 0
path = save_tu(tu)
assert os.path.exists(path)
assert os.path.getsize(path) > 0
tu2 = TranslationUnit.from_ast_file(filename=path)
assert len(tu2.diagnostics) == 0
foo = get_cursor(tu2, 'foo')
assert foo is not None
# Just in case there is an open file descriptor somewhere.
del tu2
os.unlink(path)
def _ast_files_to_callinfo(directory):
index = Index(conf.lib.clang_createIndex(False, True))
# don't list comprehend so we can get better error reporting
units = []
for path in _ast_files(directory):
try:
units.append((os.path.abspath(path),
TranslationUnit.from_ast_file(path, index)))
except Exception as e:
print("error parsing {}, python clang version might be different"
"from compiled clang version?".format(path))
print(e.args)
print(e.message)
raise
ci = CallInfo()
for path, tu in units:
for cursor in tu.cursor.get_children():
# seems hacky, probably misses c++ cases
# stuff from includes has the include's filename
if ((cursor.kind == CursorKind.VAR_DECL) and
(cursor.location.file.name == tu.spelling)):
ci.add_global(cursor)
for path, tu in units:
# WARNING: this will fail silently and unexpectedly if
# the version of clang that generated the .ast files is
# different from the python clang library
sys.stderr.write(" processing ast file {}\n".format(path))
sys.stderr.flush()
for cursor in tu.cursor.get_children():
if (cursor.kind == CursorKind.FUNCTION_DECL or
cursor.kind == CursorKind.VAR_DECL):
decl = ci.add_decl(cursor)
ci.walk_decl(cursor, decl)
return ci
def test_code_complete_pathlike(self):
files = [(str_to_path('fake.c'), """
/// Aaa.
int test1;
/// Bbb.
void test2(void);
void f() {
}
""")]
tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
cr = tu.codeComplete(str_to_path('fake.c'), 9, 1, unsaved_files=files, include_brief_comments=True)
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} || Priority: 40 || Availability: Available || Brief comment: None"
]
self.check_completion_results(cr, expected)
