def Translate(self, pytd_file): # pylint: disable=invalid-name
"""Given an open .pytd file, return a CLIF AST ast_pb2.AST protobuffer."""
pb = ast_pb2.AST()
pb.source = pytd_file.name
for hdr in self._scan_includes:
pb.usertype_includes.append(hdr)
self._include(0, [hdr], pb, scan_only=True)
if self._preamble:
self._Parse(self._preamble, pb)
self._Parse(pytd_file.read(), pb)
# Since most C++ calls release GIL we need threads.
pb.extra_init.append('PyEval_InitThreads();')
# Assumed we always want a deterministic output proto. Since dict/set order
# is not, do sorted() order traversal.
order = sorted
for t in order(self._typetable):
tm = pb.typemaps.add()
tm.lang_type = t
for ct in order(set(self._typetable[t])):
tm.cpp_type.append(ct)
for t in order(self._capsules):
tm = pb.typemaps.add()
tm.lang_type = t
tm.cpp_type.append(self._capsules[t])
for name, (nargs, src, local) in order(self._macros.items()):
if local:
m = pb.macros.add()
m.name = name
# Use protocol 0 to keep version independence.
m.definition = pickle.dumps(
(sys.hexversion, pytd_parser.version, (nargs, src)), 0)
return pb
def test_complement_ast(self):
file_content = """
namespace a {
class A {
public:
virtual int func1() = 0;
virtual int func2();
};
}; // namespace a
"""
module = extractor.Module.from_source(file_content)
ast = ast_pb2.AST()
class_decl = ast_pb2.Decl()
class_decl.decltype = ast_pb2.Decl.Type.CLASS
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.name.cpp_name = '::a::A::func1'
class_decl.class_.members.append(func_decl)
func_decl = ast_pb2.Decl()
func_decl.decltype = ast_pb2.Decl.Type.FUNC
func_decl.func.name.cpp_name = '::a::A::func2'
class_decl.class_.members.append(func_decl)
ast.decls.append(class_decl)
ast = extractor._complement_matcher_ast(ast, module)
ast_str = self._format_ast_proto(ast)
self.assertIn('::a::A::func1:is_pure_virtual:True', ast_str)
self.assertIn('::a::A::func2:is_pure_virtual:False', ast_str)
def test_complement_ast(self):
file_content = """
namespace a {
class A {
public:
virtual int func1() = 0;
virtual int func1(int a);
virtual int func2();
};
}; // namespace a
"""
module = extractor.Module.from_source(file_content)
ast = ast_pb2.AST()
class_decl = ast_pb2.Decl()
class_decl.decltype = ast_pb2.Decl.Type.CLASS
class_decl.class_.members.append(gen_func_proto(
'::a::A::func1', 'int'))
class_decl.class_.members.append(
gen_func_proto('::a::A::func1', 'int', ['int']))
class_decl.class_.members.append(gen_func_proto(
'::a::A::func2', 'int'))
ast.decls.append(class_decl)
extractor._complement_matcher_ast(ast, module)
ast_str = self._format_ast_proto(ast)
self.assertIn('::a::A::func1:0:is_pure_virtual:True', ast_str)
self.assertIn('::a::A::func1:1:is_pure_virtual:False', ast_str)
self.assertIn('::a::A::func2:0:is_pure_virtual:False', ast_str)
def testProtoHolderByValue(self):
ast = ast_pb2.AST()
ast.source = 'depot'
ph = shared_ptr_proto_member.ProtoHolderByValue(ast)
self.assertEqual(ph.GetByValue().source, 'depot')
self.assertEqual(ph.GetConstRef().source, 'depot')
self.assertIsNone(ph.ResetSource('pool'))
self.assertEqual(ph.GetConstRef().source, 'pool')
self.assertEqual(ast.source, 'depot')
def testProtoHolderByValue(self, wrapper_lib):
ast = ast_pb2.AST()
ast.source = 'depot'
ph = wrapper_lib.ProtoHolderByValue(ast)
self.assertEqual(ph.GetByValue().source, 'depot')
self.assertEqual(ph.GetConstRef().source, 'depot')
self.assertIsNone(ph.ResetSource('pool'))
self.assertEqual(ph.GetConstRef().source, 'pool')
self.assertEqual(ast.source, 'depot')
def testModConst(self):
ast = ast_pb2.AST()
text_format.Parse(
"""
decls {
decltype: CONST
const {
name {
native: "ONE"
cpp_name: "kOne"
}
type {
lang_type: "int"
cpp_type: "int"
}
}
}
extra_init: "if (something_wrong) goto err;"
""", ast)
self.m.init += ast.extra_init
out = '\n'.join(
itertools.chain(
self.m.WrapConst(ast.decls[0].const, -1, ''),
self.m.GenInitFunction('my.h'),
))
self.assertMultiLineEqual(
out,
textwrap.dedent("""
static struct PyModuleDef Module = {
PyModuleDef_HEAD_INIT,
"my.test", // module name
"CLIF-generated module for my.h", // module doc
-1, // module keeps state in global variables
nullptr
};
PyObject* Init() {
PyObject* module = PyModule_Create(&Module);
if (!module) return nullptr;
if (something_wrong) goto err;
if (PyModule_AddObject(module, "ONE", Clif_PyObjFrom(static_cast<int>(kOne), {})) < 0) goto err;
return module;
err:
Py_DECREF(module);
return nullptr;
}""" if self.m.py3output else """
PyObject* Init() {
PyObject* module = Py_InitModule3("my.test", nullptr, "CLIF-generated module for my.h");
if (!module) return nullptr;
if (something_wrong) goto err;
if (PyModule_AddObject(module, "ONE", Clif_PyObjFrom(static_cast<int>(kOne), {})) < 0) goto err;
return module;
err:
return nullptr;
}"""))
def testProtoHolderUniquePtr(self):
ast = ast_pb2.AST()
ast.source = 'depot'
ph = shared_ptr_proto_member.ProtoHolderUniquePtr(ast)
self.assertEqual(ph.GetUniquePtr().source, 'depot')
# HAVE actual unique_ptr semantics returning unique_ptr from C++ to Python:
self.assertIsNone(ph.GetUniquePtr())
ph = shared_ptr_proto_member.ProtoHolderUniquePtr(ast)
self.assertIsNone(ph.ResetSource('pool'))
self.assertEqual(ph.GetUniquePtr().source, 'pool')
# NO actual unique_ptr semantics in __init__ (passing from Python to C++):
self.assertEqual(ast.source, 'depot')
def testFuncInit(self):
ast = ast_pb2.AST()
text_format.Parse(
"""
decls {
decltype: FUNC
func {
name {
native: "x"
cpp_name: "x_"
}
returns {
type {
lang_type: "int"
cpp_type: "int"
}
}
}
}
""", ast)
# Fill context.
for d in ast.decls:
list(self.m.WrapDecl(d))
out = '\n'.join(self.m.GenInitFunction('my.h'), )
if self.m.py3output:
self.assertMultiLineEqual(
out,
textwrap.dedent(r"""
static struct PyModuleDef Module = {
PyModuleDef_HEAD_INIT,
"my.test", // module name
"CLIF-generated module for my.h", // module doc
-1, // module keeps state in global variables
Methods
};
PyObject* Init() {
PyObject* module = PyModule_Create(&Module);
if (!module) return nullptr;
return module;
}"""))
else:
self.assertMultiLineEqual(
out,
textwrap.dedent(r"""
PyObject* Init() {
PyObject* module = Py_InitModule3("my.test", Methods, "CLIF-generated module for my.h");
if (!module) return nullptr;
return module;
}"""))
def testModConst(self):
ast = ast_pb2.AST()
text_format.Parse(
"""
decls {
decltype: CONST
const {
name {
native: "ONE"
cpp_name: "kOne"
}
type {
lang_type: "int"
cpp_type: "int"
}
}
}
extra_init: "if (something_wrong) goto err;"
""", ast)
self.m.init += ast.extra_init
out = '\n'.join(
itertools.chain(
self.m.WrapConst(ast.decls[0].const, -1, ''),
self.m.GenInitFunction('my.h'),
))
# pylint: disable=g-long-ternary
self.assertMultiLineEqual(
out,
textwrap.dedent("""
static struct PyModuleDef Module = {
PyModuleDef_HEAD_INIT,
ThisModuleName,
"CLIF-generated module for my.h", // module doc
-1, // module keeps state in global variables
nullptr,
nullptr, // m_slots a.k.a. m_reload
nullptr, // m_traverse
ClearImportCache // m_clear
};
PyObject* Init() {
PyObject* module = PyModule_Create(&Module);
if (!module) return nullptr;
if (something_wrong) goto err;
if (PyModule_AddObject(module, "ONE", Clif_PyObjFrom(static_cast<int>(kOne), {})) < 0) goto err;
return module;
err:
Py_DECREF(module);
return nullptr;
}"""))
def _RunMatcher(command, data):
"""Spawn backend process to process data and capture output."""
ast = ast_pb2.AST() # Matcher output.
# Debug print(' '.join(command))
mrun = subprocess.Popen(command, stdin=PIPE, stdout=PIPE)
astpb, e = mrun.communicate(data)
rc = mrun.returncode
if rc:
raise _BackendError('Matcher failed with status %s' % rc)
if e:
raise _BackendError(e)
if not astpb:
raise _BackendError('Matcher failed with empty output')
ast.ParseFromString(astpb)
return ast
def testProtoHolderSharedPtr(self):
ast = ast_pb2.AST()
ast.source = 'depot'
ph = shared_ptr_proto_member.ProtoHolderSharedPtr(ast)
self.assertEqual(ph.GetSharedPtrUseCount(), 1)
sp1 = ph.GetSharedPtr()
# NO actual shared_ptr semantics returning shared_ptr from C++ to Python:
self.assertEqual(ph.GetSharedPtrUseCount(), 1)
self.assertEqual(sp1.source, 'depot')
self.assertIsNone(ph.ResetSource('pool'))
sp2 = ph.GetSharedPtr()
self.assertEqual(sp2.source, 'pool')
# NO actual shared_ptr semantics in __init__ (passing from Python to C++):
self.assertEqual(ast.source, 'depot')
# Each shared_ptr return actually makes a new copy of the proto:
self.assertEqual(sp1.source, 'depot')
def testFuncInit(self):
ast = ast_pb2.AST()
text_format.Parse(
"""
decls {
decltype: FUNC
func {
name {
native: "x"
cpp_name: "x_"
}
returns {
type {
lang_type: "int"
cpp_type: "int"
}
}
}
}
""", ast)
# Fill context.
for d in ast.decls:
list(self.m.WrapDecl(d))
out = '\n'.join(self.m.GenInitFunction('my.h'), )
self.assertMultiLineEqual(
out,
textwrap.dedent(r"""
static struct PyModuleDef Module = {
PyModuleDef_HEAD_INIT,
ThisModuleName,
"CLIF-generated module for my.h", // module doc
-1, // module keeps state in global variables
MethodsStaticAlloc,
nullptr, // m_slots a.k.a. m_reload
nullptr, // m_traverse
ClearImportCache // m_clear
};
PyObject* Init() {
PyObject* module = PyModule_Create(&Module);
if (!module) return nullptr;
return module;
}"""))
def testProtoParam(self):
pb = ast_pb2.AST()
self.assertEqual(t4.Walk(pb), 0)
self.assertEqual(t4.Size(pb), 0)
def testProtoParam(self, wrapper_lib):
pb = ast_pb2.AST()
self.assertEqual(wrapper_lib.Walk(pb), 0)
self.assertEqual(wrapper_lib.Size(pb), 0)
