def test_map_gcc5(self):
"""
The code in test_map_gcc5.hpp was breaking pygccxml.
Test that case (gcc5 + castxml + c++11).
See issue #45 and #55
"""
if self.config.xml_generator == "gccxml":
return
decls = parser.parse([self.header], self.config)
global_ns = declarations.get_global_namespace(decls)
# This calldef is defined with gcc > 4.9 (maybe earlier, not tested)
# and -std=c++11. Calling create_decl_string is failing with gcc.
# With clang the calldef does not exist so the matcher
# will just return an empty list, letting the test pass.
# See the test_argument_without_name.py for an equivalent test,
# which is not depending on the presence of the _M_clone_node
# method in the stl_tree.h file.
criteria = declarations.calldef_matcher(name="_M_clone_node")
free_funcs = declarations.matcher.find(criteria, global_ns)
for free_funcs in free_funcs:
free_funcs.create_decl_string(with_defaults=False)
def test_map_gcc5(self):
"""
The code in test_map_gcc5.hpp was breaking pygccxml.
Test that case (gcc5 + castxml + c++11).
See issue #45 and #55
"""
if self.config.xml_generator == "gccxml":
return
decls = parser.parse([self.header], self.config)
global_ns = declarations.get_global_namespace(decls)
# This calldef is defined with gcc > 4.9 (maybe earlier, not tested)
# and -std=c++11. Calling create_decl_string is failing with gcc.
# With clang the calldef does not exist so the matcher
# will just return an empty list, letting the test pass.
# See the test_argument_without_name.py for an equivalent test,
# which is not depending on the presence of the _M_clone_node
# method in the stl_tree.h file.
criteria = declarations.calldef_matcher(name="_M_clone_node")
free_funcs = declarations.matcher.find(criteria, global_ns)
for free_funcs in free_funcs:
free_funcs.create_decl_string(with_defaults=False)
def test_argument_without_name(self):
"""
Test passing an object without name to a templated function.
The test was failing when building the declaration string.
The declaration string will be 'void (*)( & )'. If the passed
object had a name the result would then be 'void (*)(Name & )'.
See bug #55
"""
if self.config.xml_generator == "gccxml":
return
decls = parser.parse([self.header], self.config)
global_ns = declarations.get_global_namespace(decls)
criteria = declarations.calldef_matcher(name="function")
free_funcs = declarations.matcher.find(criteria, global_ns)
for free_func in free_funcs:
decl_string = free_func.create_decl_string(with_defaults=False)
self.assertEqual(decl_string, "void (*)( & )")
def get_header_info(self):
"""
PyGCCXML header code parser
magic happens here!
: returns the parsed header data in python dict
: return dict keys: namespace, class, io_signature, make,
properties, methods
: Can be used as an CLI command or an external API
"""
gr = self.modname.split('-')[0]
module = self.modname.split('-')[-1]
self.parsed_data['module_name'] = module
generator_path, generator_name = utils.find_xml_generator()
xml_generator_config = parser.xml_generator_configuration_t(
xml_generator_path=generator_path,
xml_generator=generator_name,
include_paths=self.include_paths,
compiler='gcc',
define_symbols=['BOOST_ATOMIC_DETAIL_EXTRA_BACKEND_GENERIC'],
cflags='-std=c++11')
decls = parser.parse([self.target_file], xml_generator_config)
global_namespace = declarations.get_global_namespace(decls)
# namespace
try:
self.parsed_data['namespace'] = []
ns = global_namespace.namespace(gr)
if ns is None:
raise BlockToolException
main_namespace = ns.namespace(module)
if main_namespace is None:
raise BlockToolException('namespace cannot be none')
self.parsed_data['namespace'] = [gr, module]
if main_namespace.declarations:
for _namespace in main_namespace.declarations:
if isinstance(_namespace, declarations.namespace_t):
if Constants.KERNEL not in str(_namespace):
main_namespace = _namespace
self.parsed_data['namespace'].append(
str(_namespace).split('::')[-1].split(' ')[0])
except RuntimeError:
raise BlockToolException(
'Invalid namespace format in the block header file')
# class
try:
self.parsed_data['class'] = ''
for _class in main_namespace.declarations:
if isinstance(_class, declarations.class_t):
expected_class_name = self.filename.split('.')[0]
if expected_class_name in str(_class):
main_class = _class
self.parsed_data['class'] = str(_class).split(
'::')[2].split(' ')[0]
# in more complicated blocks, there are many classes included in this declaration
# Break after the first class - safe to assume this is the "main class"?
if len(main_class.bases) > 0:
self.parsed_data['block_type'] = main_class.bases[
0].declaration_path[-1]
break
except RuntimeError:
raise BlockToolException(
'Block header namespace {} must consist of a valid class instance'
.format(module))
# io_signature, message_ports
self.parsed_data['io_signature'] = {}
self.parsed_data['message_port'] = {}
if os.path.isfile(self.impl_file) and exist_comments(self):
self.parsed_data['io_signature'] = io_signature(self.impl_file)
self.parsed_data['message_port'] = message_port(self.impl_file)
read_comments(self)
elif os.path.isfile(self.impl_file) and not exist_comments(self):
self.parsed_data['io_signature'] = io_signature(self.impl_file)
self.parsed_data['message_port'] = message_port(self.impl_file)
if self.addcomments:
add_comments(self)
elif not os.path.isfile(self.impl_file) and exist_comments(self):
read_comments(self)
else:
self.parsed_data['io_signature'] = {"input": [], "output": []}
self.parsed_data['message_port'] = self.parsed_data['io_signature']
# make
try:
self.parsed_data['make'] = {}
self.parsed_data['make']['arguments'] = []
query_m = declarations.custom_matcher_t(
lambda mem_fun: mem_fun.name.startswith('make'))
query_make = query_m & declarations.access_type_matcher_t('public')
make_func = main_class.member_functions(
function=query_make,
allow_empty=True,
header_file=self.target_file)
criteria = declarations.calldef_matcher(name='make')
_make_fun = declarations.matcher.get_single(criteria, main_class)
_make_fun = str(_make_fun).split('make')[-1].split(')')[0].split(
'(')[1].lstrip().rstrip().split(',')
if make_func:
for arg in make_func[0].arguments:
make_arguments = None
'''
for _arg in _make_fun:
if str(arg.name) in _arg:
make_arguments = {
"name": str(arg.name),
"dtype": str(arg.decl_type),
"default": ""
}
if re.findall(r'[-+]?\d*\.\d+|\d+', _arg):
make_arguments['default'] = re.findall(
r'[-+]?\d*\.\d+|\d+', _arg)[0]
elif re.findall(r'\"(.+?)\"', _arg):
make_arguments['default'] = re.findall(
r'\"(.+?)\"', _arg)[0]
elif "true" in _arg:
make_arguments['default'] = "True"
elif "false" in _arg:
make_arguments['default'] = "False"
'''
# In case the search did not find an argument in the inner loop
# This happens while parsing digital/symbol_sync_cc.h
if make_arguments:
self.parsed_data['make']['arguments'].append(
make_arguments.copy())
else:
self.parsed_data['make']['arguments'].append({
"name":
str(arg.name),
"dtype":
str(arg.decl_type),
"default":
arg.
default_value # can we get default argument directly from arg
})
except RuntimeError:
self.parsed_data['make'] = {}
self.parsed_data['make']['arguments'] = []
# setters
try:
self.parsed_data['methods'] = []
query_methods = declarations.access_type_matcher_t('public')
setters = main_class.member_functions(function=query_methods,
allow_empty=True,
header_file=self.target_file)
getter_arguments = []
if setters:
for setter in setters:
if str(setter.name).startswith(
'set_') and setter.arguments:
setter_args = {
"name": str(setter.name),
"arguments_type": []
}
for argument in setter.arguments:
args = {
"name": str(argument.name),
"dtype": str(argument.decl_type)
}
getter_arguments.append(args['name'])
setter_args['arguments_type'].append(args.copy())
self.parsed_data['methods'].append(setter_args.copy())
except RuntimeError:
self.parsed_data['methods'] = []
# getters
try:
self.parsed_data['properties'] = []
query_properties = declarations.access_type_matcher_t('public')
getters = main_class.member_functions(function=query_properties,
allow_empty=True,
header_file=self.target_file)
if getters:
for getter in getters:
if not getter.arguments or getter.has_const:
getter_args = {
"name": str(getter.name),
"dtype": str(getter.return_type),
"read_only": True
}
if getter_args['name'] in getter_arguments:
getter_args["read_only"] = False
self.parsed_data['properties'].append(
getter_args.copy())
except RuntimeError:
self.parsed_data['properties'] = []
# all member functions
# setters and getters do not return all member functions for a block
try:
self.parsed_data['member_functions'] = []
query_methods = declarations.access_type_matcher_t('public')
functions = main_class.member_functions(
function=query_methods,
allow_empty=True,
header_file=self.target_file)
if functions:
for fcn in functions:
if str(fcn.name) not in [
main_class.name, '~' + main_class.name, 'make'
]:
fcn_args = {"name": str(fcn.name), "arguments": []}
for argument in fcn.arguments:
args = {
"name": str(argument.name),
"dtype": str(argument.decl_type),
"default": argument.default_value
}
fcn_args['arguments'].append(args.copy())
self.parsed_data['member_functions'].append(
fcn_args.copy())
except RuntimeError:
self.parsed_data['member_functions'] = []
# documentation
try:
_index = None
header_file = codecs.open(self.target_file, 'r', 'cp932')
self.parsed_data['docstring'] = re.compile(
r'//.*?$|/\*.*?\*/',
re.DOTALL | re.MULTILINE).findall(header_file.read())[2:]
header_file.close()
for doc in self.parsed_data['docstring']:
if Constants.BLOCKTOOL in doc:
_index = self.parsed_data['docstring'].index(doc)
if _index is not None:
self.parsed_data['docstring'] = self.parsed_data[
'docstring'][:_index]
except:
self.parsed_data['docstring'] = []
return self.parsed_data
# The c++ file we want to parse
filename = "example.hpp"
filename = this_module_dir_path + "/" + filename
# Parse the c++ file
decls = parser.parse([filename], xml_generator_config)
global_namespace = declarations.get_global_namespace(decls)
ns_namespace = global_namespace.namespace("ns")
int_type = declarations.cpptypes.int_t()
double_type = declarations.cpptypes.double_t()
# Search for the function by name
criteria = declarations.calldef_matcher(name="func1")
func1 = declarations.matcher.get_single(criteria, ns_namespace)
# Search for functions which return an int
criteria = declarations.calldef_matcher(return_type="int")
func2 = declarations.matcher.get_single(criteria, ns_namespace)
# Search for functions which return an int, using the cpptypes class
criteria = declarations.calldef_matcher(return_type=int_type)
func3 = declarations.matcher.get_single(criteria, ns_namespace)
print(func1)
print(func2)
print(func3)
# This prints:
def parse_file(filename):
# Find the location of the xml generator (castxml or gccxml)
generator_path, generator_name = utils.find_xml_generator(name='castxml')
# Configure the xml generator
xml_generator_config = parser.xml_generator_configuration_t(
# cflags="-std=gnu++11",
cflags="-Wno-c++11-extensions",
include_paths=["/Users/markoates/Repos/allegro_flare/include"],
xml_generator_path=generator_path,
compiler="g++",
xml_generator=generator_name)
# Parse the c++ file
decls = parser.parse([filename], xml_generator_config)
# output some GOLD!
global_namespace = declarations.get_global_namespace(decls)
# Search for functions which return a double. Two functions will be found
criteria = declarations.calldef_matcher(
#return_type="float",
#header_file="/Users/markoates/Repos/allegro_flare/include/allegro_flare/" + filename)
header_file=os.path.abspath(filename))
# criteria = declarations.calldef_matcher(return_type=double_type)
found_items = declarations.matcher.find(criteria, global_namespace)
# populate the table with unique names
count = 0
for item in found_items:
count = count + 1
cleaned_filename = re.match(r'/Users/markoates/Repos/allegro_flare/(.*)', item.location.file_name).group(1)
# create `declaration_type`
declaration_type = item.__class__.__name__
if declaration_type[-2:] == "_t":
declaration_type = declaration_type[:-2]
declaration_type = declaration_type.replace('_', ' ')
declaration_type = "%s" % declaration_type
# create `declaration`
declaration = str(item)
declaration = re.match(r'(.*) \[.*\]$', declaration).group(1)
# try to extract any preceeding '//' comments above the declaration
in_source_documentation = extract_in_source_documentation(item.location.file_name, item.location.line)
# get grandparent's name
grandparent = item.parent.parent
grandparent_name = grandparent.name if grandparent else ''
# insert the content into the table
parse_cache_make_table_connection.execute("INSERT INTO parsed_declarations VALUES (NULL,?,?,?,?,?,?,?,?,?,?,?,?);",
(str(item), item.decl_string, declaration, item.name, declaration_type, str(item.parent.name), str(grandparent_name), item.attributes, cleaned_filename, str(item.location.line), in_source_documentation, "")
)
parse_cache_connection.commit()
# create a list of unique items
unique_item_names = set();
for item in found_items:
unique_item_names.update({item.name})
# cross-correlate declarations in the database
docs_connection = sqlite3.connect('doc_entries.db')
docs_connection.row_factory = sqlite3.Row
docs_c = docs_connection.cursor()
found_items = 0
unfound_items = 0
for item in unique_item_names:
docs_c.execute('SELECT * FROM entries WHERE decl=?', (item, ))
entries = docs_c.fetchall()
if len(entries) == 0:
print item
unfound_items += 1
else:
print_green(item + " - FOUND")
found_items += 1
if unfound_items == 0:
print_func = print_green
elif found_items == 0:
print_func = print_red
else:
print_func = print_yellow
print_func("==============================")
print_func(str(found_items) + " items found.")
print_func(str(unfound_items) + " matches missing.")
print_func("==============================")
return
xml_generator_config = parser.xml_generator_configuration_t(
xml_generator_path=generator_path, xml_generator=generator_name)
# The c++ file we want to parse
filename = "example.hpp"
filename = this_module_dir_path + "/" + filename
# Parse the c++ file
decls = parser.parse([filename], xml_generator_config)
global_namespace = declarations.get_global_namespace(decls)
ns_namespace = global_namespace.namespace("ns")
# Search for the function called func1
criteria = declarations.calldef_matcher(name="func1")
func1a = declarations.matcher.get_single(criteria, ns_namespace)
# Search for the function called func2
criteria = declarations.calldef_matcher(name="func2")
func2a = declarations.matcher.get_single(criteria, ns_namespace)
# You can also write a loop on the declaration tree
func1b = None
for decl in ns_namespace.declarations:
if decl.name == "func1":
func1b = decl
# The declarations can be compared (prints (True, False))
print(func1a == func1b, func1a == func2a)
def get_header_info(self):
"""
PyGCCXML header code parser
magic happens here!
: returns the parsed header data in python dict
: return dict keys: namespace, class, io_signature, make,
properties, methods
: Can be used as an CLI command or an extenal API
"""
gr = self.modname.split('-')[0]
module = self.modname.split('-')[-1]
generator_path, generator_name = utils.find_xml_generator()
xml_generator_config = parser.xml_generator_configuration_t(
xml_generator_path=generator_path,
xml_generator=generator_name,
compiler='gcc')
decls = parser.parse(
[self.target_file], xml_generator_config)
global_namespace = declarations.get_global_namespace(decls)
# namespace
try:
self.parsed_data['namespace'] = []
ns = global_namespace.namespace(gr)
if ns is None:
raise BlockToolException
main_namespace = ns.namespace(module)
if main_namespace is None:
raise BlockToolException('namespace cannot be none')
self.parsed_data['namespace'] = [gr, module]
if main_namespace.declarations:
for _namespace in main_namespace.declarations:
if isinstance(_namespace, declarations.namespace_t):
if Constants.KERNEL not in str(_namespace):
main_namespace = _namespace
self.parsed_data['namespace'].append(
str(_namespace).split('::')[-1].split(' ')[0])
except RuntimeError:
raise BlockToolException(
'Invalid namespace format in the block header file')
# class
try:
self.parsed_data['class'] = ''
for _class in main_namespace.declarations:
if isinstance(_class, declarations.class_t):
main_class = _class
self.parsed_data['class'] = str(_class).split('::')[
2].split(' ')[0]
except RuntimeError:
raise BlockToolException(
'Block header namespace {} must consist of a valid class instance'.format(module))
# io_signature, message_ports
self.parsed_data['io_signature'] = {}
self.parsed_data['message_port'] = {}
if os.path.isfile(self.impl_file) and exist_comments(self):
self.parsed_data['io_signature'] = io_signature(
self.impl_file)
self.parsed_data['message_port'] = message_port(
self.impl_file)
read_comments(self)
elif os.path.isfile(self.impl_file) and not exist_comments(self):
self.parsed_data['io_signature'] = io_signature(
self.impl_file)
self.parsed_data['message_port'] = message_port(
self.impl_file)
if self.addcomments:
add_comments(self)
elif not os.path.isfile(self.impl_file) and exist_comments(self):
read_comments(self)
else:
self.parsed_data['io_signature'] = {
"input": [],
"output": []
}
self.parsed_data['message_port'] = self.parsed_data['io_signature']
# make
try:
self.parsed_data['make'] = {}
self.parsed_data['make']['arguments'] = []
query_m = declarations.custom_matcher_t(
lambda mem_fun: mem_fun.name.startswith('make'))
query_make = query_m & declarations.access_type_matcher_t('public')
make_func = main_class.member_functions(function=query_make,
allow_empty=True,
header_file=self.target_file)
criteria = declarations.calldef_matcher(name='make')
_make_fun = declarations.matcher.get_single(criteria, main_class)
_make_fun = str(_make_fun).split(
'make')[-1].split(')')[0].split('(')[1].lstrip().rstrip().split(',')
if make_func:
for arg in make_func[0].arguments:
for _arg in _make_fun:
if str(arg.name) in _arg:
make_arguments = {
"name": str(arg.name),
"dtype": str(arg.decl_type),
"default": ""
}
if re.findall(r'[-+]?\d*\.\d+|\d+', _arg):
make_arguments['default'] = re.findall(
r'[-+]?\d*\.\d+|\d+', _arg)[0]
elif re.findall(r'\"(.+?)\"', _arg):
make_arguments['default'] = re.findall(
r'\"(.+?)\"', _arg)[0]
elif "true" in _arg:
make_arguments['default'] = "True"
elif "false" in _arg:
make_arguments['default'] = "False"
self.parsed_data['make']['arguments'].append(
make_arguments.copy())
except RuntimeError:
self.parsed_data['make'] = {}
self.parsed_data['make']['arguments'] = []
# setters
try:
self.parsed_data['methods'] = []
query_methods = declarations.access_type_matcher_t('public')
setters = main_class.member_functions(function=query_methods,
allow_empty=True,
header_file=self.target_file)
getter_arguments = []
if setters:
for setter in setters:
if str(setter.name).startswith('set_') and setter.arguments:
setter_args = {
"name": str(setter.name),
"arguments_type": []
}
for argument in setter.arguments:
args = {
"name": str(argument.name),
"dtype": str(argument.decl_type)
}
getter_arguments.append(args['name'])
setter_args['arguments_type'].append(args.copy())
self.parsed_data['methods'].append(setter_args.copy())
except RuntimeError:
self.parsed_data['methods'] = []
# getters
try:
self.parsed_data['properties'] = []
query_properties = declarations.access_type_matcher_t('public')
getters = main_class.member_functions(function=query_properties,
allow_empty=True,
header_file=self.target_file)
if getters:
for getter in getters:
if not getter.arguments or getter.has_const:
getter_args = {
"name": str(getter.name),
"dtype": str(getter.return_type),
"read_only": True
}
if getter_args['name'] in getter_arguments:
getter_args["read_only"] = False
self.parsed_data['properties'].append(
getter_args.copy())
except RuntimeError:
self.parsed_data['properties'] = []
# documentation
try:
_index = None
header_file = codecs.open(self.target_file, 'r', 'cp932')
self.parsed_data['docstring'] = re.compile(
r'//.*?$|/\*.*?\*/', re.DOTALL | re.MULTILINE).findall(
header_file.read())[2:]
header_file.close()
for doc in self.parsed_data['docstring']:
if Constants.BLOCKTOOL in doc:
_index = self.parsed_data['docstring'].index(doc)
if _index is not None:
self.parsed_data['docstring'] = self.parsed_data['docstring'][: _index]
except:
self.parsed_data['docstring'] = []
return self.parsed_data
def parse_file(filename):
# Find the location of the xml generator (castxml or gccxml)
generator_path, generator_name = utils.find_xml_generator(name='castxml')
# Configure the xml generator
xml_generator_config = parser.xml_generator_configuration_t(
# cflags="-std=gnu++11",
cflags="-Wno-c++11-extensions",
include_paths=["/Users/markoates/Repos/allegro_flare/include"],
xml_generator_path=generator_path,
compiler="g++",
xml_generator=generator_name)
# Parse the c++ file
decls = parser.parse([filename], xml_generator_config)
# output some GOLD!
global_namespace = declarations.get_global_namespace(decls)
# Search for functions which return a double. Two functions will be found
criteria = declarations.calldef_matcher(
#return_type="float",
#header_file="/Users/markoates/Repos/allegro_flare/include/allegro_flare/" + filename)
header_file=os.path.abspath(filename))
# criteria = declarations.calldef_matcher(return_type=double_type)
found_items = declarations.matcher.find(criteria, global_namespace)
# populate the table with unique names
count = 0
for item in found_items:
count = count + 1
#print item.location.file_name + " : " + str(item.location.line)
cleaned_filename = re.match(
r'/Users/markoates/Repos/allegro_flare/(.*)',
item.location.file_name).group(1)
# create `declaration_type`
declaration_type = item.__class__.__name__
if declaration_type[-2:] == "_t":
declaration_type = declaration_type[:-2]
declaration_type = declaration_type.replace('_', ' ')
declaration_type = "%s" % declaration_type
# create `declaration`
declaration = str(item)
declaration = re.match(r'(.*) \[.*\]$', declaration).group(1)
parse_cache_make_table_connection.execute(
"INSERT INTO parsed_declarations VALUES (NULL,?,?,?,?,?,?,?,?,?,?,?,?);",
(str(item), item.decl_string, declaration, item.name,
declaration_type, str(item.parent.name), str(
item.top_parent.name), item.attributes, cleaned_filename,
str(item.location.line), "", ""))
parse_cache_connection.commit()
# create a list of unique items
unique_item_names = set()
for item in found_items:
unique_item_names.update({item.name})
# cross-correlate declarations in the database
docs_connection = sqlite3.connect('doc_entries.db')
docs_connection.row_factory = sqlite3.Row
docs_c = docs_connection.cursor()
found_items = 0
unfound_items = 0
for item in unique_item_names:
docs_c.execute('SELECT * FROM entries WHERE decl=?', (item, ))
entries = docs_c.fetchall()
if len(entries) == 0:
print item
unfound_items += 1
else:
print_green(item + " - FOUND")
found_items += 1
if unfound_items == 0:
print_func = print_green
elif found_items == 0:
print_func = print_red
else:
print_func = print_yellow
print_func("==============================")
print_func(str(found_items) + " items found.")
print_func(str(unfound_items) + " matches missing.")
print_func("==============================")
return
xml_generator_path=generator_path,
xml_generator=generator_name)
# The c++ file we want to parse
filename = "example.hpp"
filename = this_module_dir_path + "/" + filename
# Parse the c++ file
decls = parser.parse([filename], xml_generator_config)
global_namespace = declarations.get_global_namespace(decls)
ns_namespace = global_namespace.namespace("ns")
# Search for the function called func1
criteria = declarations.calldef_matcher(name="func1")
func1a = declarations.matcher.get_single(criteria, ns_namespace)
# Search for the function called func2
criteria = declarations.calldef_matcher(name="func2")
func2a = declarations.matcher.get_single(criteria, ns_namespace)
# You can also write a loop on the declaration tree
func1b = None
for decl in ns_namespace.declarations:
if decl.name == "func1":
func1b = decl
# The declarations can be compared (prints (True, False))
print(func1a == func1b, func1a == func2a)