def run(self): print('Processing "%s" ..' % self.filename, file=sys.stderr) try: index = cindex.Index(cindex.conf.lib.clang_createIndex( False, True)) tu = index.parse(self.filename, self.parameters) extract(self.filename, tu.cursor, '', self.output) finally: job_semaphore.release()
def run(self): global errors_detected print('Processing "%s" ..' % self.filename, file=sys.stderr) try: index = cindex.Index( cindex.conf.lib.clang_createIndex(False, True)) tu = index.parse(self.filename, self.parameters) extract(self.filename, tu.cursor, '', self.output) except BaseException: errors_detected = True raise finally: job_semaphore.release()
def run(self): print('Processing "%s" ..' % self.filename, file=sys.stderr) context = { "filename": self.filename, "parameters": self.parameters, "classes": {}, "class_templates": {}, "class_template_instantiations": {}, } index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) tu = index.parse(self.filename, self.parameters) extract_nodes(self.filename, tu.cursor, self.nodes) for n in self.nodes: # dump_node(n) p = getparent(n, self.nodes) if p is None or p.kind == CursorKind.NAMESPACE or p.kind == CursorKind.TRANSLATION_UNIT: if n.kind in NODE_HANDLERS.keys(): func = NODE_HANDLERS[n.kind] func(n, self.nodes, self.output, ' ', 'm', context)
def main(): parameters = ['-x', 'c++', '-D__MKDOC_PY__'] filenames = [] library_file = None if platform.system() == 'Darwin': completed_process = subprocess.run(['xcrun', '--find', 'clang'], stdout=subprocess.PIPE, encoding='utf-8') if completed_process.returncode == 0: toolchain_dir = os.path.dirname( os.path.dirname(completed_process.stdout.strip())) library_file = os.path.join(toolchain_dir, 'lib', 'libclang.dylib') completed_process = subprocess.run(['xcrun', '--show-sdk-path'], stdout=subprocess.PIPE, encoding='utf-8') if completed_process.returncode == 0: sdkroot = completed_process.stdout.strip() if os.path.exists(sdkroot): parameters.append('-isysroot') parameters.append(sdkroot) elif platform.system() == 'Linux': library_file = '/usr/lib/llvm-6.0/lib/libclang.so' if library_file and os.path.exists(library_file): cindex.Config.set_library_path(os.path.dirname(library_file)) quiet = False std = '-std=c++11' root_name = 'mkdoc_doc' ignore_patterns = [] output_filename = None for item in sys.argv[1:]: if item == '-quiet': quiet = True elif item.startswith('-output='): output_filename = item[len('-output='):] elif item.startswith('-std='): std = item elif item.startswith('-root-name='): root_name = item[len('-root-name='):] elif item.startswith('-exclude-hdr-patterns='): ignore_patterns.append(item[len('-exclude-hdr-patterns='):]) elif item.startswith('-'): parameters.append(item) else: filenames.append(item) parameters.append(std) if output_filename is None or len(filenames) == 0: eprint('Syntax: %s -output=<file> [.. a list of header files ..]' % sys.argv[0]) sys.exit(1) f = open(output_filename, 'w', encoding='utf-8') # N.B. We substitute the `GENERATED FILE...` bits in this fashion because # otherwise Reviewable gets confused. f.write('''#pragma once // {0} {1} // This file contains docstrings for the Python bindings that were // automatically extracted by mkdoc.py. #if defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #endif '''.format('GENERATED FILE', 'DO NOT EDIT')) # Determine project include directories. # N.B. For simplicity when using with Bazel, we do not try to get canonical # file paths for determining include files. include_paths = [] for param in parameters: # Only check for normal include directories. if param.startswith("-I"): include_paths.append(param[2:]) # Use longest include directories first to get shortest include file # overall. include_paths = list(sorted(include_paths, key=len))[::-1] include_files = [] # Create mapping from filename to include file. include_file_map = FileDict() for filename in filenames: for include_path in include_paths: prefix = include_path + "/" if filename.startswith(prefix): include_file = filename[len(prefix):] break else: raise RuntimeError( "Filename not incorporated into -I includes: {}".format( filename)) for p in ignore_patterns: if fnmatch(include_file, p): break else: include_files.append(include_file) include_file_map[filename] = include_file assert len(include_files) > 0 # Generate the glue include file, which will include all relevant include # files, and parse. Use a tempdir that is relative to the output file for # usage with Bazel. tmpdir = output_filename + ".tmp_artifacts" os.mkdir(tmpdir) glue_filename = os.path.join(tmpdir, "mkdoc_glue.h") with open(glue_filename, 'w') as glue_f: for include_file in sorted(include_files): line = "#include \"{}\"".format(include_file) glue_f.write(line + "\n") f.write("// " + line + "\n") f.write("\n") glue_f.flush() if not quiet: eprint("Parse headers...") index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) translation_unit = index.parse( glue_filename, parameters, options=cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD) shutil.rmtree(tmpdir) # Extract symbols. if not quiet: eprint("Extract relevant symbols...") symbol_tree = SymbolTree() extract(include_file_map, translation_unit.cursor, symbol_tree) # Write header file. if not quiet: eprint("Writing header file...") try: print_symbols(f, root_name, symbol_tree.root) except UnicodeEncodeError as e: # User-friendly error for #9903. print(""" Encountered unicode error: {} If you are on Ubuntu, please ensure you have en_US.UTF-8 locales generated: sudo apt-get install --no-install-recommends locales sudo locale-gen en_US.UTF-8 """.format(e), file=sys.stderr) sys.exit(1) f.write(''' #if defined(__GNUG__) #pragma GCC diagnostic pop #endif ''')
def main(): parameters = ['-x', 'c++', '-D__MKDOC_PY__'] filenames = [] if platform.system() == 'Darwin': dev_path = '/Applications/Xcode.app/Contents/Developer/' lib_dir = dev_path + 'Toolchains/XcodeDefault.xctoolchain/usr/lib/' sdk_dir = dev_path + 'Platforms/MacOSX.platform/Developer/SDKs' libclang = lib_dir + 'libclang.dylib' if os.path.exists(libclang): cindex.Config.set_library_path(os.path.dirname(libclang)) if os.path.exists(sdk_dir): sysroot_dir = os.path.join(sdk_dir, next(os.walk(sdk_dir))[1][0]) parameters.append('-isysroot') parameters.append(sysroot_dir) quiet = False std = '-std=c++11' root_name = 'mkdoc_doc' ignore_patterns = [] output_filename = None for item in sys.argv[1:]: if item == '-quiet': quiet = True elif item.startswith('-output='): output_filename = item[len('-output='):] elif item.startswith('-std='): std = item elif item.startswith('-root-name='): root_name = item[len('-root-name='):] elif item.startswith('-exclude-hdr-patterns='): ignore_patterns.append(item[len('-exclude-hdr-patterns='):]) elif item.startswith('-'): parameters.append(item) else: filenames.append(item) parameters.append(std) if output_filename is None or len(filenames) == 0: eprint('Syntax: %s -output=<file> [.. a list of header files ..]' % sys.argv[0]) sys.exit(1) f = open(output_filename, 'w') # N.B. We substitute the `GENERATED FILE...` bits in this fashion because # otherwise Reviewable gets confused. f.write('''#pragma once // {0} {1} // This file contains docstrings for the Python bindings that were // automatically extracted by mkdoc.py. #if defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #endif '''.format('GENERATED FILE', 'DO NOT EDIT')) # Determine project include directories. # N.B. For simplicity when using with Bazel, we do not try to get canonical # file paths for determining include files. include_paths = [] for param in parameters: # Only check for normal include directories. if param.startswith("-I"): include_paths.append(param[2:]) # Use longest include directories first to get shortest include file # overall. include_paths = list(sorted(include_paths, key=len))[::-1] include_files = [] # Create mapping from filename to include file. include_file_map = FileDict() for filename in filenames: for include_path in include_paths: prefix = include_path + "/" if filename.startswith(prefix): include_file = filename[len(prefix):] break else: raise RuntimeError( "Filename not incorporated into -I includes: {}".format( filename)) for p in ignore_patterns: if fnmatch(include_file, p): break else: include_files.append(include_file) include_file_map[filename] = include_file assert len(include_files) > 0 # Generate the glue include file, which will include all relevant include # files, and parse. Add a unique prefix so we do not leak accidentally leak # in paths in `/tmp`. dir_prefix = mkdtemp(prefix="drake_mkdoc_") glue_include_file = NamedTemporaryFile('w', prefix="glue_include_file_", dir=dir_prefix) with glue_include_file: for include_file in sorted(include_files): line = "#include \"{}\"".format(include_file) glue_include_file.write(line + "\n") f.write("// " + line + "\n") f.write("\n") glue_include_file.flush() if not quiet: eprint("Parse headers...") index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) translation_unit = index.parse(glue_include_file.name, parameters) os.rmdir(dir_prefix) # Extract symbols. if not quiet: eprint("Extract relevant symbols...") symbol_tree = SymbolTree() extract(include_file_map, translation_unit.cursor, symbol_tree) # Write header file. if not quiet: eprint("Writing header file...") try: print_symbols(f, root_name, symbol_tree.root) except UnicodeEncodeError as e: # User-friendly error for #9903. print(""" Encountered unicode error: {} If you are on Ubuntu, please ensure you have en_US.UTF-8 locales generated: sudo apt-get install --no-install-recommends locales sudo locale-gen en_US.UTF-8 """.format(e), file=sys.stderr) sys.exit(1) f.write(''' #if defined(__GNUG__) #pragma GCC diagnostic pop #endif ''')
def main(): parameters = ['-x', 'c++', '-D__MKDOC_PY__'] add_library_paths(parameters) filenames = [] quiet = False std = '-std=c++11' root_name = 'mkdoc_doc' ignore_patterns = [] output_filename = None output_filename_xml = None # TODO(m-chaturvedi): Consider using argparse. for item in sys.argv[1:]: if item == '-quiet': quiet = True elif item.startswith('-output='): output_filename = item[len('-output='):] elif item.startswith('-output_xml='): output_filename_xml = item[len('-output_xml='):] elif item.startswith('-std='): std = item elif item.startswith('-ignore-dirs-for-coverage='): ignore_dir_str = item[len('-ignore-dirs-for-coverage='):] ignore_dirs_for_coverage = None if ignore_dir_str: ignore_dirs_for_coverage = tuple(ignore_dir_str.split(',')) elif item.startswith('-root-name='): root_name = item[len('-root-name='):] elif item.startswith('-exclude-hdr-patterns='): ignore_patterns.append(item[len('-exclude-hdr-patterns='):]) elif item.startswith('-'): parameters.append(item) else: filenames.append(item) parameters.append(std) if output_filename is None or len(filenames) == 0: eprint('Syntax: %s -output=<file> [.. a list of header files ..]' % sys.argv[0]) sys.exit(1) f = open(output_filename, 'w', encoding='utf-8') f_xml = None if output_filename_xml is not None: f_xml = open(output_filename_xml, 'w') # N.B. We substitute the `GENERATED FILE...` bits in this fashion because # otherwise Reviewable gets confused. f.write('''#pragma once // {0} {1} // This file contains docstrings for the Python bindings that were // automatically extracted by mkdoc.py. #if defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #endif '''.format('GENERATED FILE', 'DO NOT EDIT')) # Determine project include directories. # N.B. For simplicity when using with Bazel, we do not try to get canonical # file paths for determining include files. include_paths = [] for param in parameters: # Only check for normal include directories. if param.startswith("-I"): include_paths.append(param[2:]) # Use longest include directories first to get shortest include file # overall. include_paths = list(sorted(include_paths, key=len))[::-1] include_files = [] # Create mapping from filename to include file. include_file_map = FileDict() used_ignore_patterns = set() for filename in filenames: for include_path in include_paths: prefix = include_path + "/" if filename.startswith(prefix): include_file = filename[len(prefix):] break else: raise RuntimeError( "Filename not incorporated into -I includes: {}".format( filename)) for p in ignore_patterns: if fnmatch(include_file, p): used_ignore_patterns.add(p) break else: include_files.append(include_file) include_file_map[filename] = include_file assert len(include_files) > 0 unused_ignore_patterns = set(ignore_patterns) - used_ignore_patterns if unused_ignore_patterns: print(f"Unused ignore patterns: {unused_ignore_patterns}") # Generate the glue include file, which will include all relevant include # files, and parse. Use a tempdir that is relative to the output file for # usage with Bazel. tmpdir = output_filename + ".tmp_artifacts" os.mkdir(tmpdir) glue_filename = os.path.join(tmpdir, "mkdoc_glue.h") with open(glue_filename, 'w') as glue_f: for include_file in sorted(include_files): line = "#include \"{}\"".format(include_file) glue_f.write(line + "\n") f.write("// " + line + "\n") f.write("\n") glue_f.flush() if not quiet: eprint("Parse headers...") index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) translation_unit = index.parse( glue_filename, parameters, options=cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD) if not translation_unit: raise RuntimeError( "Parsing headers using the clang library failed") severities = [ diagnostic.severity for diagnostic in translation_unit.diagnostics if diagnostic.severity >= cindex.Diagnostic.Error ] if severities: raise RuntimeError( ("Parsing headers using the clang library failed with {} " "error(s) and {} fatal error(s)").format( severities.count(cindex.Diagnostic.Error), severities.count(cindex.Diagnostic.Fatal))) shutil.rmtree(tmpdir) # Extract symbols. if not quiet: eprint("Extract relevant symbols...") symbol_tree = SymbolTree() extract(include_file_map, translation_unit.cursor, symbol_tree) # Write header file. if not quiet: eprint("Writing header file...") try: tree_parser = { "tree_parser_doc": [], "tree_parser_xpath": [ET.Element("Root")], "ignore_dirs_for_coverage": ignore_dirs_for_coverage } print_symbols(f, root_name, symbol_tree.root, **tree_parser) except UnicodeEncodeError as e: # User-friendly error for #9903. print(""" Encountered unicode error: {} If you are on Ubuntu, please ensure you have en_US.UTF-8 locales generated: sudo apt-get install --no-install-recommends locales sudo locale-gen en_US.UTF-8 """.format(e), file=sys.stderr) sys.exit(1) f.write(''' #if defined(__GNUG__) #pragma GCC diagnostic pop #endif ''') if f_xml is not None: f_xml.write(prettify(tree_parser["tree_parser_xpath"][0]))
for i in node.get_children(): extract_nodes(filename, i, output, sub_prefix) if node.kind in PRINT_LIST: output.append(node) else: # print("Ignored: {}".format(node.kind)) pass return output def dump_node(n, prefix=''): print(prefix + "name: {0:40} {2:20} displayname: {1}".format( n.spelling, n.displayname, str(n.kind).replace('CursorKind.', ''))) def dump(all): i = 0 for node in all: dump_node(node, "{:3} ".format(i)) i += 1 index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) tu = index.parse(filename, parameters) nodes = extract_nodes(filename, tu.cursor) #s = [x for x in nodes if x.kind==CursorKind.STRUCT_DECL] #s0=s[0] #s1=s[1] dump(nodes)