def test_complement_ast(self): file_content = """ namespace a { class A { public: virtual int func1() = 0; virtual int func2(); }; }; // namespace a """ module = extractor.Module.from_source(file_content) ast = ast_pb2.AST() class_decl = ast_pb2.Decl() class_decl.decltype = ast_pb2.Decl.Type.CLASS func_decl = ast_pb2.Decl() func_decl.decltype = ast_pb2.Decl.Type.FUNC func_decl.func.name.cpp_name = '::a::A::func1' class_decl.class_.members.append(func_decl) func_decl = ast_pb2.Decl() func_decl.decltype = ast_pb2.Decl.Type.FUNC func_decl.func.name.cpp_name = '::a::A::func2' class_decl.class_.members.append(func_decl) ast.decls.append(class_decl) ast = extractor._complement_matcher_ast(ast, module) ast_str = self._format_ast_proto(ast) self.assertIn('::a::A::func1:is_pure_virtual:True', ast_str) self.assertIn('::a::A::func2:is_pure_virtual:False', ast_str)
def MoveExtendPropertiesInPlace(ast): """See module docstring.""" extend_property_decls = [] extend_getter_decls = [] for decl in ast.decls: member_delete_indices = [] if decl.decltype != ast_pb2.Decl.Type.CLASS: continue for member_index, member in enumerate(decl.class_.members): if member.decltype != ast_pb2.Decl.Type.VAR: continue if not member.var.is_extend_variable: continue property_decl = ast_pb2.Decl() property_decl.CopyFrom(member) property_decl.var.name.native = (decl.class_.name.native + EXTEND_INFIX + member.var.name.native) if member.var.name.cpp_name == member.var.name.native: property_decl.var.name.cpp_name = (decl.class_.name.native + EXTEND_INFIX + member.var.name.cpp_name) p = _GenerateParameterSelf(decl.class_) property_decl.var.cpp_get.params.insert(0, p) property_decl.var.cpp_get.name.cpp_name = ( decl.class_.name.native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name) if member.var.HasField('cpp_set'): property_decl.var.cpp_set.name.cpp_name = ( decl.class_.name.native + EXTEND_INFIX + member.var.cpp_set.name.cpp_name) p = _GenerateParameterSelf(decl.class_) property_decl.var.cpp_set.params.insert(0, p) extend_property_decls.append(property_decl) member_delete_indices.append(member_index) # generate property getters # (setters do not need this kind of functionality) getter_decl = ast_pb2.FuncDecl() getter_decl.CopyFrom(member.var.cpp_get) getter_decl.name.native = getter_decl.name.cpp_name = ( decl.class_.name.native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name) p = _GenerateParameterSelf(decl.class_) getter_decl.params.insert(0, p) getter_decl.is_extend_method = True func_decl = ast_pb2.Decl() func_decl.func.CopyFrom(getter_decl) func_decl.decltype = ast_pb2.Decl.Type.FUNC extend_getter_decls.append(func_decl) for member_index in reversed(member_delete_indices): del decl.class_.members[member_index] ast.decls.extend(extend_property_decls) ast.decls.extend(extend_getter_decls)
def _MoveExtendPropertiesInPlaceOneClass( extend_property_decls, extend_getter_decls, outer_class_names, class_decl): """Helper for MoveExtendPropertiesInPlace.""" member_delete_indices = [] for member_index, member in enumerate(class_decl.members): if member.decltype == ast_pb2.Decl.Type.CLASS: _MoveExtendPropertiesInPlaceOneClass( extend_property_decls, extend_getter_decls, outer_class_names+[class_decl.name], member.class_) if member.decltype != ast_pb2.Decl.Type.VAR: continue if not member.var.is_extend_variable: continue fq_native = '_'.join( [n.native for n in outer_class_names + [class_decl.name]]) property_decl = ast_pb2.Decl() property_decl.CopyFrom(member) property_decl.var.name.native = ( fq_native + EXTEND_INFIX + member.var.name.native) if member.var.name.cpp_name == member.var.name.native: property_decl.var.name.cpp_name = ( fq_native + EXTEND_INFIX + member.var.name.cpp_name) p = _GenerateParameterSelf(class_decl, outer_class_names) property_decl.var.cpp_get.params.insert(0, p) property_decl.var.cpp_get.name.cpp_name = ( fq_native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name) if member.var.HasField('cpp_set'): property_decl.var.cpp_set.name.cpp_name = ( fq_native + EXTEND_INFIX + member.var.cpp_set.name.cpp_name) p = _GenerateParameterSelf(class_decl, outer_class_names) property_decl.var.cpp_set.params.insert(0, p) extend_property_decls.append(property_decl) member_delete_indices.append(member_index) # generate property getters # (setters do not need this kind of functionality) getter_decl = ast_pb2.FuncDecl() getter_decl.CopyFrom(member.var.cpp_get) getter_decl.name.native = getter_decl.name.cpp_name = ( fq_native + EXTEND_INFIX + member.var.cpp_get.name.cpp_name) p = _GenerateParameterSelf(class_decl, outer_class_names) getter_decl.params.insert(0, p) getter_decl.is_extend_method = True func_decl = ast_pb2.Decl() func_decl.func.CopyFrom(getter_decl) func_decl.decltype = ast_pb2.Decl.Type.FUNC extend_getter_decls.append(func_decl) for member_index in reversed(member_delete_indices): del class_decl.members[member_index]
def test_complement_ast(self): file_content = """ namespace a { class A { public: virtual int func1() = 0; virtual int func1(int a); virtual int func2(); }; }; // namespace a """ module = extractor.Module.from_source(file_content) ast = ast_pb2.AST() class_decl = ast_pb2.Decl() class_decl.decltype = ast_pb2.Decl.Type.CLASS class_decl.class_.members.append(gen_func_proto( '::a::A::func1', 'int')) class_decl.class_.members.append( gen_func_proto('::a::A::func1', 'int', ['int'])) class_decl.class_.members.append(gen_func_proto( '::a::A::func2', 'int')) ast.decls.append(class_decl) extractor._complement_matcher_ast(ast, module) ast_str = self._format_ast_proto(ast) self.assertIn('::a::A::func1:0:is_pure_virtual:True', ast_str) self.assertIn('::a::A::func1:1:is_pure_virtual:False', ast_str) self.assertIn('::a::A::func2:0:is_pure_virtual:False', ast_str)
def _MoveOutOfClassScope(param0_name_native, class_decl, orig_func_decl): """Move extended method declaration out of its class declaration.""" func_decl = ast_pb2.Decl() func_decl.CopyFrom(orig_func_decl) if orig_func_decl.func.constructor: extend_infix = EXTEND_INFIX_CONSTRUCTOR else: extend_infix = EXTEND_INFIX name_native = (class_decl.class_.name.native + extend_infix + orig_func_decl.func.name.native) name_cpp_name = (class_decl.class_.name.native + extend_infix + orig_func_decl.func.name.cpp_name) func_decl.func.name.native = name_native if orig_func_decl.func.name.cpp_name == orig_func_decl.func.name.native: func_decl.func.name.cpp_name = name_cpp_name elif orig_func_decl.func.constructor: # cpp_name of constructors is equal to the class name. We need to change it # to be the extended function name. func_decl.func.name.cpp_name = func_decl.func.name.native if (func_decl.func.params and func_decl.func.params[0].name.native == param0_name_native): # A fully-qualified cpp_name was specified. del func_decl.func.params[0] return func_decl
def MoveExtendPropertiesBackIntoClassesInPlace(ast): """See module docstring.""" extend_properties_orig_decl_indices = [] extracted_property_decls_by_class_name = collections.defaultdict(list) for orig_decl_index, decl in enumerate(ast.decls): if decl.decltype != ast_pb2.Decl.Type.VAR: continue if not decl.var.is_extend_variable: continue class_name, property_name = decl.var.name.native.split(EXTEND_INFIX, 1) property_decl = ast_pb2.Decl() property_decl.CopyFrom(decl) property_decl.var.name.native = property_name if property_decl.var.HasField('cpp_set'): del property_decl.var.cpp_set.params[0] extend_properties_orig_decl_indices.append(orig_decl_index) extracted_property_decls_by_class_name[class_name].append( property_decl) for orig_decl_index in reversed(extend_properties_orig_decl_indices): del ast.decls[orig_decl_index] for target_decl in ast.decls: if target_decl.decltype != ast_pb2.Decl.Type.CLASS: continue class_name = target_decl.class_.name.native extracted_property_decls = extracted_property_decls_by_class_name.get( class_name) if extracted_property_decls is None: continue for extracted_decl in extracted_property_decls: target_decl.class_.members.append(extracted_decl) del extracted_property_decls_by_class_name[class_name] assert not extracted_property_decls_by_class_name
def MoveExtendFunctionsBackIntoClassesInPlace( ast, class_decls_by_fq_native, omit_self): """See module docstring.""" extend_methods_orig_decl_indices = [] for orig_decl_index, decl in enumerate(ast.decls): if decl.decltype != ast_pb2.Decl.Type.FUNC: continue if not decl.func.is_extend_method: continue if not decl.func.classmethod and not decl.func.constructor: assert decl.func.params, 'extended method does not have any parameters' assert decl.func.params[0].name.native == 'self', ( 'the first parameter of extended method `%s` is not `self`' % decl.func.name.native) if decl.func.constructor: extend_infix = EXTEND_INFIX_CONSTRUCTOR else: extend_infix = EXTEND_INFIX fq_native_from_func, method_name_from_func = decl.func.name.native.split( extend_infix, 1) target_class_decl = class_decls_by_fq_native[fq_native_from_func] method_decl = ast_pb2.Decl() method_decl.CopyFrom(decl) method_decl.func.name.native = method_name_from_func if omit_self and not decl.func.classmethod and not decl.func.constructor: # Explicit self is needed for the PyCLIF code generator, # but confuses pytype. del method_decl.func.params[0] target_class_decl.members.append(method_decl) extend_methods_orig_decl_indices.append(orig_decl_index) for orig_decl_index in reversed(extend_methods_orig_decl_indices): del ast.decls[orig_decl_index]
def _MoveOutOfClassScope( param0_name_native, class_decl, orig_func_decl, outer_class_names): """Helper for _MoveExtendMethodsToFunctionsOneClass.""" func_decl = ast_pb2.Decl() func_decl.CopyFrom(orig_func_decl) fq_native = '_'.join( [n.native for n in outer_class_names + [class_decl.name]]) if orig_func_decl.func.constructor: extend_infix = EXTEND_INFIX_CONSTRUCTOR else: extend_infix = EXTEND_INFIX name_native = fq_native + extend_infix + orig_func_decl.func.name.native name_cpp_name = fq_native + extend_infix + orig_func_decl.func.name.cpp_name func_decl.func.name.native = name_native if orig_func_decl.func.name.cpp_name == orig_func_decl.func.name.native: func_decl.func.name.cpp_name = name_cpp_name elif orig_func_decl.func.constructor: # cpp_name of constructors is equal to the class name. We need to change it # to be the extended function name. func_decl.func.name.cpp_name = func_decl.func.name.native if (func_decl.func.params and func_decl.func.params[0].name.native == param0_name_native): # A fully-qualified cpp_name was specified. del func_decl.func.params[0] return func_decl
def unproperty(self, ln, ast, class_name, members, known_names): """Translate PYTD func IR into VAR getter / setter protobuf. Args: ln: .clif line number ast: func IR class_name: Python name of the wrapped C++ class members: wrapped C++ class memebers known_names: names already defined in a C++ class wrapper Returns: True if ast is a @getter/@setter func that describes 'unproperty' C++ var. Raises: ValueError: if @getter or @setter refers to exising var with =property(). """ assert ast[0] == 'func', repr(ast) getset = ast.decorators.asList() if getset not in (['getter'], ['setter']): return False # Convert func to var. f = ast_pb2.Decl() self._func(ln, ast, f) f = f.func cname = f.name.cpp_name pyname = f.name.native for m in members: if m.decltype == m.VAR and m.var.name.cpp_name == cname: if m.var.cpp_get.name.cpp_name: raise ValueError( "property var can't have @getter / @setter func") if m.var.name.native == pyname: known_names.discard(m.var.name.native) break else: m = members.add() m.line_number = ln m.decltype = m.VAR m.var.name.cpp_name = cname _add_uniq(class_name, known_names, pyname) p = m.var if getset == ['getter']: if len(f.returns) != 1 or f.params: raise TypeError('@getter signature must be (self)->T') p.type.CopyFrom(f.returns[0].type) p.cpp_get.name.native = pyname else: assert getset == ['setter'] if len(f.params) != 1 or f.returns: raise TypeError('@setter signature must be (self, _:T)') p.type.CopyFrom(f.params[0].type) p.cpp_set.name.native = pyname if not p.cpp_get.name.native: # Provide default getter as VARNAME(). p.cpp_get.name.native = cname # Preserve func param varname for the error message (not useful otherwise) p.name.native = f.params[0].name.native return True
def gen_func_proto(name, return_type=None, param_types=None): func_decl = ast_pb2.Decl() func_decl.decltype = ast_pb2.Decl.Type.FUNC func_decl.func.name.cpp_name = name if return_type: func_decl.func.returns.append( ast_pb2.ParamDecl(cpp_exact_type=return_type)) if param_types: for param_type in param_types: func_decl.func.params.append( ast_pb2.ParamDecl(cpp_exact_type=param_type)) return func_decl
def MoveExtendFunctionsBackIntoClassesInPlace(ast, omit_self=False): """See module docstring.""" extend_methods_orig_decl_indices = [] extracted_method_decls_by_class_name = collections.defaultdict(list) for orig_decl_index, decl in enumerate(ast.decls): if decl.decltype != ast_pb2.Decl.Type.FUNC: continue if not decl.func.is_extend_method: continue if not decl.func.classmethod and not decl.func.constructor: assert decl.func.params, 'extended method does not have any parameters' assert decl.func.params[0].name.native == 'self', ( 'the first parameter of extended method `%s` is not `self`' % decl.func.name.native) if decl.func.constructor: extend_infix = EXTEND_INFIX_CONSTRUCTOR else: extend_infix = EXTEND_INFIX class_name_from_func, method_name_from_func = decl.func.name.native.split( extend_infix, 1) method_decl = ast_pb2.Decl() method_decl.CopyFrom(decl) method_decl.func.name.native = method_name_from_func if omit_self and not decl.func.classmethod and not decl.func.constructor: # Explicit self is needed for the PyCLIF code generator, # but confuses pytype. del method_decl.func.params[0] extend_methods_orig_decl_indices.append(orig_decl_index) extracted_method_decls_by_class_name[class_name_from_func].append( method_decl) for orig_decl_index in reversed(extend_methods_orig_decl_indices): del ast.decls[orig_decl_index] for target_decl in ast.decls: if target_decl.decltype != ast_pb2.Decl.Type.CLASS: continue class_name = target_decl.class_.name.native extracted_method_decls = extracted_method_decls_by_class_name.get( class_name) if extracted_method_decls is None: continue for extracted_decl in extracted_method_decls: target_decl.class_.members.append(extracted_decl) del extracted_method_decls_by_class_name[class_name] assert not extracted_method_decls_by_class_name
def MoveExtendPropertiesBackIntoClassesInPlace(ast, class_decls_by_fq_native): """See module docstring.""" extend_properties_orig_decl_indices = [] for orig_decl_index, decl in enumerate(ast.decls): if decl.decltype != ast_pb2.Decl.Type.VAR: continue if not decl.var.is_extend_variable: continue fq_native_from_var, property_name_from_var = decl.var.name.native.split( EXTEND_INFIX, 1) target_class_decl = class_decls_by_fq_native[fq_native_from_var] property_decl = ast_pb2.Decl() property_decl.CopyFrom(decl) property_decl.var.name.native = property_name_from_var if property_decl.var.HasField('cpp_set'): del property_decl.var.cpp_set.params[0] target_class_decl.members.append(property_decl) extend_properties_orig_decl_indices.append(orig_decl_index) for orig_decl_index in reversed(extend_properties_orig_decl_indices): del ast.decls[orig_decl_index]
def _traverse_class(self, cursor: Cursor, namespaces: List[Text]) -> None: """Traverses a class in clang AST and registers its declaration. This method modifies self._class_decl_map. Args: cursor: The cursor which is pointing to the head of a class in clang AST. namespaces: The namespace in which the class is in. """ class_decl = ast_pb2.Decl() class_decl.decltype = ast_pb2.Decl.Type.CLASS namespace = self._gen_namespace_str(namespaces) fully_qualified_name = '::'.join([namespace, cursor.spelling]) for c in cursor.get_children(): if (c.kind == CursorKind.CXX_METHOD or c.kind == CursorKind.CONSTRUCTOR and c.type.kind == TypeKind.FUNCTIONPROTO): child_namespaces = list(namespaces) child_namespaces.append(cursor.spelling) self._traverse_function(c, child_namespaces) self._class_decl_map[fully_qualified_name] = class_decl
def _traverse_function(self, cursor: Cursor, namespaces: List[Text]) -> ast_pb2.Decl: """Traverses a function in clang AST and registers its declaration. This method modifies self._func_decl_map. Args: cursor: The cursor which is pointing to the head of a function in clang AST. namespaces: The namespace in which the function is in. Returns: The function declaration. This is needed so that the class decl which defines this function can add it as a member. """ func_decl = ast_pb2.Decl() func_decl.decltype = ast_pb2.Decl.Type.FUNC func_decl.func.is_pure_virtual = cursor.is_pure_virtual_method() namespace = self._gen_namespace_str(namespaces) fully_qualified_name = '::'.join([namespace, cursor.spelling]) self._func_decl_map[fully_qualified_name] = func_decl return func_decl
def testReturnSmartPtr(self): t4.GetUniquePtr(ast_pb2.Decl()) t4.GetSharedPtr(ast_pb2.Decl())
def testWrongMessageType(self): with self.assertRaises(TypeError): t4.Walk(ast_pb2.Decl())
def testProtoNestedParam(self): pb = ast_pb2.Decl() pb.decltype = pb.FUNC self.assertEqual(t4.DeclType(pb), pb.FUNC) self.assertEqual(t4.DeclTypeUI(pb), pb.FUNC) self.assertEqual(t4.DeclTypeUO(pb), pb.FUNC)
def testReturnSmartPtr(self, wrapper_lib): wrapper_lib.GetUniquePtr(ast_pb2.Decl()) wrapper_lib.GetSharedPtr(ast_pb2.Decl())
def testWrongMessageType(self, wrapper_lib): with self.assertRaises(TypeError): wrapper_lib.Walk(ast_pb2.Decl())
def testProtoNestedParam(self, wrapper_lib): pb = ast_pb2.Decl() pb.decltype = pb.FUNC self.assertEqual(wrapper_lib.DeclType(pb), pb.FUNC) self.assertEqual(wrapper_lib.DeclTypeUI(pb), pb.FUNC) self.assertEqual(wrapper_lib.DeclTypeUO(pb), pb.FUNC)