def test_complement_ast(self):
file_content = """
namespace a {
class A {
public:
virtual int func1() = 0;
virtual int func2();
};
}; // namespace a
"""
module = extractor.Module.from_source(file_content)
ast = ast_pb2.AST()
class_decl = ast_pb2.Decl()
class_decl.decltype = ast_pb2.Decl.Type.CLASS
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.name.cpp_name = '::a::A::func1'
class_decl.class_.members.append(func_decl)
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.name.cpp_name = '::a::A::func2'
class_decl.class_.members.append(func_decl)
ast.decls.append(class_decl)
ast = extractor._complement_matcher_ast(ast, module)
ast_str = self._format_ast_proto(ast)
self.assertIn('::a::A::func1:is_pure_virtual:True', ast_str)
self.assertIn('::a::A::func2:is_pure_virtual:False', ast_str)
def MoveExtendPropertiesInPlace(ast):
"""See module docstring."""
extend_property_decls = []
extend_getter_decls = []
for decl in ast.decls:
member_delete_indices = []
if decl.decltype != ast_pb2.Decl.Type.CLASS:
continue
for member_index, member in enumerate(decl.class_.members):
if member.decltype != ast_pb2.Decl.Type.VAR:
continue
if not member.var.is_extend_variable:
continue
property_decl = ast_pb2.Decl()
property_decl.CopyFrom(member)
property_decl.var.name.native = (decl.class_.name.native +
EXTEND_INFIX +
member.var.name.native)
if member.var.name.cpp_name == member.var.name.native:
property_decl.var.name.cpp_name = (decl.class_.name.native +
EXTEND_INFIX +
member.var.name.cpp_name)
p = _GenerateParameterSelf(decl.class_)
property_decl.var.cpp_get.params.insert(0, p)
property_decl.var.cpp_get.name.cpp_name = (
decl.class_.name.native + EXTEND_INFIX +
member.var.cpp_get.name.cpp_name)
if member.var.HasField('cpp_set'):
property_decl.var.cpp_set.name.cpp_name = (
decl.class_.name.native + EXTEND_INFIX +
member.var.cpp_set.name.cpp_name)
p = _GenerateParameterSelf(decl.class_)
property_decl.var.cpp_set.params.insert(0, p)
extend_property_decls.append(property_decl)
member_delete_indices.append(member_index)
# generate property getters
# (setters do not need this kind of functionality)
getter_decl = ast_pb2.FuncDecl()
getter_decl.CopyFrom(member.var.cpp_get)
getter_decl.name.native = getter_decl.name.cpp_name = (
decl.class_.name.native + EXTEND_INFIX +
member.var.cpp_get.name.cpp_name)
p = _GenerateParameterSelf(decl.class_)
getter_decl.params.insert(0, p)
getter_decl.is_extend_method = True
func_decl = ast_pb2.Decl()
func_decl.func.CopyFrom(getter_decl)
func_decl.decltype = ast_pb2.Decl.Type.FUNC
extend_getter_decls.append(func_decl)
for member_index in reversed(member_delete_indices):
del decl.class_.members[member_index]
ast.decls.extend(extend_property_decls)
ast.decls.extend(extend_getter_decls)
def _MoveExtendPropertiesInPlaceOneClass(
extend_property_decls, extend_getter_decls, outer_class_names, class_decl):
"""Helper for MoveExtendPropertiesInPlace."""
member_delete_indices = []
for member_index, member in enumerate(class_decl.members):
if member.decltype == ast_pb2.Decl.Type.CLASS:
_MoveExtendPropertiesInPlaceOneClass(
extend_property_decls, extend_getter_decls,
outer_class_names+[class_decl.name], member.class_)
if member.decltype != ast_pb2.Decl.Type.VAR:
continue
if not member.var.is_extend_variable:
continue
fq_native = '_'.join(
[n.native for n in outer_class_names + [class_decl.name]])
property_decl = ast_pb2.Decl()
property_decl.CopyFrom(member)
property_decl.var.name.native = (
fq_native + EXTEND_INFIX + member.var.name.native)
if member.var.name.cpp_name == member.var.name.native:
property_decl.var.name.cpp_name = (
fq_native + EXTEND_INFIX + member.var.name.cpp_name)
p = _GenerateParameterSelf(class_decl, outer_class_names)
property_decl.var.cpp_get.params.insert(0, p)
property_decl.var.cpp_get.name.cpp_name = (
fq_native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name)
if member.var.HasField('cpp_set'):
property_decl.var.cpp_set.name.cpp_name = (
fq_native + EXTEND_INFIX + member.var.cpp_set.name.cpp_name)
p = _GenerateParameterSelf(class_decl, outer_class_names)
property_decl.var.cpp_set.params.insert(0, p)
extend_property_decls.append(property_decl)
member_delete_indices.append(member_index)
# generate property getters
# (setters do not need this kind of functionality)
getter_decl = ast_pb2.FuncDecl()
getter_decl.CopyFrom(member.var.cpp_get)
getter_decl.name.native = getter_decl.name.cpp_name = (
fq_native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name)
p = _GenerateParameterSelf(class_decl, outer_class_names)
getter_decl.params.insert(0, p)
getter_decl.is_extend_method = True
func_decl = ast_pb2.Decl()
func_decl.func.CopyFrom(getter_decl)
func_decl.decltype = ast_pb2.Decl.Type.FUNC
extend_getter_decls.append(func_decl)
for member_index in reversed(member_delete_indices):
del class_decl.members[member_index]
def test_complement_ast(self):
file_content = """
namespace a {
class A {
public:
virtual int func1() = 0;
virtual int func1(int a);
virtual int func2();
};
}; // namespace a
"""
module = extractor.Module.from_source(file_content)
ast = ast_pb2.AST()
class_decl = ast_pb2.Decl()
class_decl.decltype = ast_pb2.Decl.Type.CLASS
class_decl.class_.members.append(gen_func_proto(
'::a::A::func1', 'int'))
class_decl.class_.members.append(
gen_func_proto('::a::A::func1', 'int', ['int']))
class_decl.class_.members.append(gen_func_proto(
'::a::A::func2', 'int'))
ast.decls.append(class_decl)
extractor._complement_matcher_ast(ast, module)
ast_str = self._format_ast_proto(ast)
self.assertIn('::a::A::func1:0:is_pure_virtual:True', ast_str)
self.assertIn('::a::A::func1:1:is_pure_virtual:False', ast_str)
self.assertIn('::a::A::func2:0:is_pure_virtual:False', ast_str)
def _MoveOutOfClassScope(param0_name_native, class_decl, orig_func_decl):
"""Move extended method declaration out of its class declaration."""
func_decl = ast_pb2.Decl()
func_decl.CopyFrom(orig_func_decl)
if orig_func_decl.func.constructor:
extend_infix = EXTEND_INFIX_CONSTRUCTOR
else:
extend_infix = EXTEND_INFIX
name_native = (class_decl.class_.name.native + extend_infix +
orig_func_decl.func.name.native)
name_cpp_name = (class_decl.class_.name.native + extend_infix +
orig_func_decl.func.name.cpp_name)
func_decl.func.name.native = name_native
if orig_func_decl.func.name.cpp_name == orig_func_decl.func.name.native:
func_decl.func.name.cpp_name = name_cpp_name
elif orig_func_decl.func.constructor:
# cpp_name of constructors is equal to the class name. We need to change it
# to be the extended function name.
func_decl.func.name.cpp_name = func_decl.func.name.native
if (func_decl.func.params
and func_decl.func.params[0].name.native == param0_name_native):
# A fully-qualified cpp_name was specified.
del func_decl.func.params[0]
return func_decl
def MoveExtendPropertiesBackIntoClassesInPlace(ast):
"""See module docstring."""
extend_properties_orig_decl_indices = []
extracted_property_decls_by_class_name = collections.defaultdict(list)
for orig_decl_index, decl in enumerate(ast.decls):
if decl.decltype != ast_pb2.Decl.Type.VAR:
continue
if not decl.var.is_extend_variable:
continue
class_name, property_name = decl.var.name.native.split(EXTEND_INFIX, 1)
property_decl = ast_pb2.Decl()
property_decl.CopyFrom(decl)
property_decl.var.name.native = property_name
if property_decl.var.HasField('cpp_set'):
del property_decl.var.cpp_set.params[0]
extend_properties_orig_decl_indices.append(orig_decl_index)
extracted_property_decls_by_class_name[class_name].append(
property_decl)
for orig_decl_index in reversed(extend_properties_orig_decl_indices):
del ast.decls[orig_decl_index]
for target_decl in ast.decls:
if target_decl.decltype != ast_pb2.Decl.Type.CLASS:
continue
class_name = target_decl.class_.name.native
extracted_property_decls = extracted_property_decls_by_class_name.get(
class_name)
if extracted_property_decls is None:
continue
for extracted_decl in extracted_property_decls:
target_decl.class_.members.append(extracted_decl)
del extracted_property_decls_by_class_name[class_name]
assert not extracted_property_decls_by_class_name
def MoveExtendFunctionsBackIntoClassesInPlace(
ast, class_decls_by_fq_native, omit_self):
"""See module docstring."""
extend_methods_orig_decl_indices = []
for orig_decl_index, decl in enumerate(ast.decls):
if decl.decltype != ast_pb2.Decl.Type.FUNC:
continue
if not decl.func.is_extend_method:
continue
if not decl.func.classmethod and not decl.func.constructor:
assert decl.func.params, 'extended method does not have any parameters'
assert decl.func.params[0].name.native == 'self', (
'the first parameter of extended method `%s` is not `self`' %
decl.func.name.native)
if decl.func.constructor:
extend_infix = EXTEND_INFIX_CONSTRUCTOR
else:
extend_infix = EXTEND_INFIX
fq_native_from_func, method_name_from_func = decl.func.name.native.split(
extend_infix, 1)
target_class_decl = class_decls_by_fq_native[fq_native_from_func]
method_decl = ast_pb2.Decl()
method_decl.CopyFrom(decl)
method_decl.func.name.native = method_name_from_func
if omit_self and not decl.func.classmethod and not decl.func.constructor:
# Explicit self is needed for the PyCLIF code generator,
# but confuses pytype.
del method_decl.func.params[0]
target_class_decl.members.append(method_decl)
extend_methods_orig_decl_indices.append(orig_decl_index)
for orig_decl_index in reversed(extend_methods_orig_decl_indices):
del ast.decls[orig_decl_index]
def _MoveOutOfClassScope(
param0_name_native, class_decl, orig_func_decl, outer_class_names):
"""Helper for _MoveExtendMethodsToFunctionsOneClass."""
func_decl = ast_pb2.Decl()
func_decl.CopyFrom(orig_func_decl)
fq_native = '_'.join(
[n.native for n in outer_class_names + [class_decl.name]])
if orig_func_decl.func.constructor:
extend_infix = EXTEND_INFIX_CONSTRUCTOR
else:
extend_infix = EXTEND_INFIX
name_native = fq_native + extend_infix + orig_func_decl.func.name.native
name_cpp_name = fq_native + extend_infix + orig_func_decl.func.name.cpp_name
func_decl.func.name.native = name_native
if orig_func_decl.func.name.cpp_name == orig_func_decl.func.name.native:
func_decl.func.name.cpp_name = name_cpp_name
elif orig_func_decl.func.constructor:
# cpp_name of constructors is equal to the class name. We need to change it
# to be the extended function name.
func_decl.func.name.cpp_name = func_decl.func.name.native
if (func_decl.func.params and
func_decl.func.params[0].name.native == param0_name_native):
# A fully-qualified cpp_name was specified.
del func_decl.func.params[0]
return func_decl
def unproperty(self, ln, ast, class_name, members, known_names):
"""Translate PYTD func IR into VAR getter / setter protobuf.
Args:
ln: .clif line number
ast: func IR
class_name: Python name of the wrapped C++ class
members: wrapped C++ class memebers
known_names: names already defined in a C++ class wrapper
Returns:
True if ast is a @getter/@setter func that describes 'unproperty' C++ var.
Raises:
ValueError: if @getter or @setter refers to exising var with =property().
"""
assert ast[0] == 'func', repr(ast)
getset = ast.decorators.asList()
if getset not in (['getter'], ['setter']): return False
# Convert func to var.
f = ast_pb2.Decl()
self._func(ln, ast, f)
f = f.func
cname = f.name.cpp_name
pyname = f.name.native
for m in members:
if m.decltype == m.VAR and m.var.name.cpp_name == cname:
if m.var.cpp_get.name.cpp_name:
raise ValueError(
"property var can't have @getter / @setter func")
if m.var.name.native == pyname:
known_names.discard(m.var.name.native)
break
else:
m = members.add()
m.line_number = ln
m.decltype = m.VAR
m.var.name.cpp_name = cname
_add_uniq(class_name, known_names, pyname)
p = m.var
if getset == ['getter']:
if len(f.returns) != 1 or f.params:
raise TypeError('@getter signature must be (self)->T')
p.type.CopyFrom(f.returns[0].type)
p.cpp_get.name.native = pyname
else:
assert getset == ['setter']
if len(f.params) != 1 or f.returns:
raise TypeError('@setter signature must be (self, _:T)')
p.type.CopyFrom(f.params[0].type)
p.cpp_set.name.native = pyname
if not p.cpp_get.name.native:
# Provide default getter as VARNAME().
p.cpp_get.name.native = cname
# Preserve func param varname for the error message (not useful otherwise)
p.name.native = f.params[0].name.native
return True
def gen_func_proto(name, return_type=None, param_types=None):
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.name.cpp_name = name
if return_type:
func_decl.func.returns.append(
ast_pb2.ParamDecl(cpp_exact_type=return_type))
if param_types:
for param_type in param_types:
func_decl.func.params.append(
ast_pb2.ParamDecl(cpp_exact_type=param_type))
return func_decl
def MoveExtendFunctionsBackIntoClassesInPlace(ast, omit_self=False):
"""See module docstring."""
extend_methods_orig_decl_indices = []
extracted_method_decls_by_class_name = collections.defaultdict(list)
for orig_decl_index, decl in enumerate(ast.decls):
if decl.decltype != ast_pb2.Decl.Type.FUNC:
continue
if not decl.func.is_extend_method:
continue
if not decl.func.classmethod and not decl.func.constructor:
assert decl.func.params, 'extended method does not have any parameters'
assert decl.func.params[0].name.native == 'self', (
'the first parameter of extended method `%s` is not `self`' %
decl.func.name.native)
if decl.func.constructor:
extend_infix = EXTEND_INFIX_CONSTRUCTOR
else:
extend_infix = EXTEND_INFIX
class_name_from_func, method_name_from_func = decl.func.name.native.split(
extend_infix, 1)
method_decl = ast_pb2.Decl()
method_decl.CopyFrom(decl)
method_decl.func.name.native = method_name_from_func
if omit_self and not decl.func.classmethod and not decl.func.constructor:
# Explicit self is needed for the PyCLIF code generator,
# but confuses pytype.
del method_decl.func.params[0]
extend_methods_orig_decl_indices.append(orig_decl_index)
extracted_method_decls_by_class_name[class_name_from_func].append(
method_decl)
for orig_decl_index in reversed(extend_methods_orig_decl_indices):
del ast.decls[orig_decl_index]
for target_decl in ast.decls:
if target_decl.decltype != ast_pb2.Decl.Type.CLASS:
continue
class_name = target_decl.class_.name.native
extracted_method_decls = extracted_method_decls_by_class_name.get(
class_name)
if extracted_method_decls is None:
continue
for extracted_decl in extracted_method_decls:
target_decl.class_.members.append(extracted_decl)
del extracted_method_decls_by_class_name[class_name]
assert not extracted_method_decls_by_class_name
def MoveExtendPropertiesBackIntoClassesInPlace(ast, class_decls_by_fq_native):
"""See module docstring."""
extend_properties_orig_decl_indices = []
for orig_decl_index, decl in enumerate(ast.decls):
if decl.decltype != ast_pb2.Decl.Type.VAR:
continue
if not decl.var.is_extend_variable:
continue
fq_native_from_var, property_name_from_var = decl.var.name.native.split(
EXTEND_INFIX, 1)
target_class_decl = class_decls_by_fq_native[fq_native_from_var]
property_decl = ast_pb2.Decl()
property_decl.CopyFrom(decl)
property_decl.var.name.native = property_name_from_var
if property_decl.var.HasField('cpp_set'):
del property_decl.var.cpp_set.params[0]
target_class_decl.members.append(property_decl)
extend_properties_orig_decl_indices.append(orig_decl_index)
for orig_decl_index in reversed(extend_properties_orig_decl_indices):
del ast.decls[orig_decl_index]
def _traverse_class(self, cursor: Cursor, namespaces: List[Text]) -> None:
"""Traverses a class in clang AST and registers its declaration.
This method modifies self._class_decl_map.
Args:
cursor: The cursor which is pointing to the head of a class in clang AST.
namespaces: The namespace in which the class is in.
"""
class_decl = ast_pb2.Decl()
class_decl.decltype = ast_pb2.Decl.Type.CLASS
namespace = self._gen_namespace_str(namespaces)
fully_qualified_name = '::'.join([namespace, cursor.spelling])
for c in cursor.get_children():
if (c.kind == CursorKind.CXX_METHOD
or c.kind == CursorKind.CONSTRUCTOR
and c.type.kind == TypeKind.FUNCTIONPROTO):
child_namespaces = list(namespaces)
child_namespaces.append(cursor.spelling)
self._traverse_function(c, child_namespaces)
self._class_decl_map[fully_qualified_name] = class_decl
def _traverse_function(self, cursor: Cursor,
namespaces: List[Text]) -> ast_pb2.Decl:
"""Traverses a function in clang AST and registers its declaration.
This method modifies self._func_decl_map.
Args:
cursor: The cursor which is pointing to the head of a function in clang
AST.
namespaces: The namespace in which the function is in.
Returns:
The function declaration. This is needed so that the class decl which
defines this function can add it as a member.
"""
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.is_pure_virtual = cursor.is_pure_virtual_method()
namespace = self._gen_namespace_str(namespaces)
fully_qualified_name = '::'.join([namespace, cursor.spelling])
self._func_decl_map[fully_qualified_name] = func_decl
return func_decl
def testReturnSmartPtr(self):
t4.GetUniquePtr(ast_pb2.Decl())
t4.GetSharedPtr(ast_pb2.Decl())
def testWrongMessageType(self):
with self.assertRaises(TypeError):
t4.Walk(ast_pb2.Decl())
def testProtoNestedParam(self):
pb = ast_pb2.Decl()
pb.decltype = pb.FUNC
self.assertEqual(t4.DeclType(pb), pb.FUNC)
self.assertEqual(t4.DeclTypeUI(pb), pb.FUNC)
self.assertEqual(t4.DeclTypeUO(pb), pb.FUNC)
def testReturnSmartPtr(self, wrapper_lib):
wrapper_lib.GetUniquePtr(ast_pb2.Decl())
wrapper_lib.GetSharedPtr(ast_pb2.Decl())
def testWrongMessageType(self, wrapper_lib):
with self.assertRaises(TypeError):
wrapper_lib.Walk(ast_pb2.Decl())
def testProtoNestedParam(self, wrapper_lib):
pb = ast_pb2.Decl()
pb.decltype = pb.FUNC
self.assertEqual(wrapper_lib.DeclType(pb), pb.FUNC)
self.assertEqual(wrapper_lib.DeclTypeUI(pb), pb.FUNC)
self.assertEqual(wrapper_lib.DeclTypeUO(pb), pb.FUNC)
