def __init__(self): self.bindings = [] self.wrapper_funcs = [] self.classes = {} self.namespaces = {} self.enums = {} self.parser = hdr_parser.CppHeaderParser() self.class_idx = 0
def check_module_docs(self, name): self.parser = hp.CppHeaderParser() decls = [] self.fmap = {} for hname in opencv_hdr_list: if hname.startswith("../modules/" + name): decls += self.parser.parse(hname, wmode=False) for d in decls: fname = d[0] if not fname.startswith("struct") and not fname.startswith( "class") and not fname.startswith("const"): dlist = self.fmap.get(fname, []) dlist.append(d) self.fmap[fname] = dlist self.missing_docfunc_list = [] doclist = glob.glob("../modules/" + name + "/doc/*.rst") for d in doclist: self.process_rst(d) print "\n\n########## The list of undocumented functions: ###########\n\n" misscount = 0 fkeys = sorted(self.fmap.keys()) for f in fkeys: # skip undocumented destructors if "~" in f: continue decls = self.fmap[f] fcomps = f.split(".") prefix = "" wlist_decls = [] for c in fcomps: prefix = (prefix + "." + c).lstrip(".") wlist_decls = self.whitelist.get(prefix, []) if wlist_decls == "*": break if wlist_decls == "*": continue wlist_decls = [self.decl2str(d) for d in wlist_decls] for d in decls: dstr = self.decl2str(d) # special hack for ML: skip old variants of the methods if name == "ml" and ("CvMat" in dstr): continue if dstr not in wlist_decls: misscount += 1 print "%s %s(%s)" % (d[1], d[0].replace( ".", "::"), ", ".join( [a[0] + " " + a[1] for a in d[3]])) print "\n\n\nundocumented functions in %s: %d" % (name, misscount)
def get_typelist(headers): group_map = {} import hdr_parser headers = list(headers) + hdr_parser.opencv_hdr_list headers = map(os.path.abspath, headers) headers = set(headers) parser = hdr_parser.CppHeaderParser(False) types = {} namespaces = {} groups = {} for header in sorted(headers): header = header.replace("\\", "/") print("INFO Processing header file:", header) if not header or not os.path.exists(header): continue group = "modules/" + re.match(r".*/modules/(.+)", header).group(1) # group = group_map.get(group, None) for declaration in parser.parse(header): function = TypeDef.from_declaration(declaration) if not function: continue if function.type == "func": namespaces[function.name] = function.namespace groups[function.name] = group for arg in function.args: name = "{function.name}:{arg.name}".format(**locals()).lower() type = arg.type if type: if types.get(name, type) != type: print("ERROR Inconsistent types: " + name) types[name] = type return types, namespaces, groups
def read_whitelist(self): self.whitelist = {} try: wf = open("check_docs_whitelist.txt", "rt") except IOError: return self.parser = hp.CppHeaderParser() for l in wf.readlines(): cpos = l.find("#") if cpos >= 0: l = l[:cpos] l = l.strip() if not l: continue rst_decl = None if "(" in l: l = l.replace("cv::", "") rst_decl = self.parser.parse_func_decl_no_wrap(l) fname = rst_decl[0] else: fname = l.replace("::", ".") complist = fname.split(".") prefix = "" alreadyListed = False wl = [] for c in complist: prefix = (prefix + "." + c).lstrip(".") wl = self.whitelist.get(prefix, []) if wl == "*": break if wl == "*": continue if not rst_decl: self.whitelist[fname] = "*" else: wl.append(rst_decl) self.whitelist[fname] = wl wf.close()
def gen(self, srcfiles, output_path): self.clear() self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True) # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = self.parser.parse(hdr) if len(decls) == 0: continue self.code_include.write('#include "{0}"\n'.format( hdr[hdr.rindex('src/'):])) for decl in decls: name = decl[0] if name.startswith("struct") or name.startswith("class"): # class/struct p = name.find(" ") stype = name[:p] name = name[p + 1:].strip() self.add_class(stype, name, decl) elif name.startswith("const"): # constant self.add_const(name.replace("const ", "").strip(), decl) else: # function self.add_func(decl) # step 1.5 check if all base classes exist for name, classinfo in self.classes.items(): if classinfo.base: chunks = classinfo.base.split('_') base = '_'.join(chunks) while base not in self.classes and len(chunks) > 1: del chunks[-2] base = '_'.join(chunks) if base not in self.classes: print("Generator error: unable to resolve base %s for %s" % (classinfo.base, classinfo.name)) sys.exit(-1) base_instance = self.classes[base] classinfo.base = base classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag: # doesn't work for trees(graphs) with depth > 2 self.classes[name] = classinfo # tree-based propagation of 'isalgorithm' processed = dict() def process_isalgorithm(classinfo): if classinfo.isalgorithm or classinfo in processed: return classinfo.isalgorithm res = False if classinfo.base: res = process_isalgorithm(self.classes[classinfo.base]) #assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base classinfo.isalgorithm |= res res = classinfo.isalgorithm processed[classinfo] = True return res for name, classinfo in self.classes.items(): process_isalgorithm(classinfo) # step 2: generate code for the classes and their methods classlist = list(self.classes.items()) classlist.sort() for name, classinfo in classlist: if classinfo.ismap: self.code_types.write( gen_template_map_type_cvt.substitute( modulePrefix=modulePrefix, name=name, cname=classinfo.cname)) else: if classinfo.issimple: templ = gen_template_simple_type_decl else: templ = gen_template_type_decl self.code_types.write( templ.substitute( modulePrefix=modulePrefix, name=name, wname=classinfo.wname, cname=classinfo.cname, sname=classinfo.sname, cname1=("cv::Algorithm" if classinfo.isalgorithm else classinfo.cname))) # register classes in the same order as they have been declared. # this way, base classes will be registered in Python before their derivatives. classlist1 = [(classinfo.decl_idx, name, classinfo) for name, classinfo in classlist] classlist1.sort() for decl_idx, name, classinfo in classlist1: code = classinfo.gen_code(self) self.code_types.write(code) if not classinfo.ismap: self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name, )) self.code_type_publish.write( "PUBLISH_OBJECT(\"{name}\", {modulePrefix}_{name}_Type);\n" .format(name=classinfo.name, modulePrefix=modulePrefix)) # step 3: generate the code for all the global functions for ns_name, ns in sorted(self.namespaces.items()): #Chandra disabled #if ns_name.split('.')[0] != 'cv': # continue for name, func in sorted(ns.funcs.items()): if func.isconstructor: continue code = func.gen_code(self) self.code_funcs.write(code) self.gen_namespace(ns_name) self.gen_namespaces_reg() # step 4: generate the code for constants constlist = list(self.consts.items()) constlist.sort() for name, constinfo in constlist: self.gen_const_reg(constinfo) # That's it. Now save all the files self.save(output_path, modulePrefix + "_generated_include.h", self.code_include) self.save(output_path, modulePrefix + "_generated_funcs.h", self.code_funcs) self.save(output_path, modulePrefix + "_generated_types.h", self.code_types) self.save(output_path, modulePrefix + "_generated_type_reg.h", self.code_type_reg) self.save(output_path, modulePrefix + "_generated_ns_reg.h", self.code_ns_reg) self.save(output_path, modulePrefix + "_generated_type_publish.h", self.code_type_publish) self.save_json(output_path, modulePrefix + "_signatures.json", self.py_signatures)
if len(sys.argv) < 2: print "Usage:\n", os.path.basename(sys.argv[0]), " <module path>" exit(0) if len(sys.argv) >= 3: if sys.argv[2].lower() == "verbose": verbose = True rst_parser_dir = os.path.dirname(os.path.abspath(sys.argv[0])) hdr_parser_path = os.path.join(rst_parser_dir, "../../python/src2") sys.path.append(hdr_parser_path) import hdr_parser module = sys.argv[1] if module != "all" and not os.path.isdir(os.path.join(rst_parser_dir, "../../" + module)): print "RST parser error E%03d: module \"%s\" could not be found." % (ERROR_010_NOMODULE, module) exit(1) parser = RstParser(hdr_parser.CppHeaderParser()) if module == "all": for m in allmodules: parser.parse(m, os.path.join(rst_parser_dir, "../../" + m)) else: parser.parse(module, os.path.join(rst_parser_dir, "../../" + module)) # summary parser.printSummary()
def gen(self, srcfiles, output_path): self.clear() self.parser = hdr_parser.CppHeaderParser() # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = self.parser.parse(hdr) if len(decls) == 0: continue self.code_include.write( '#include "{0}"\n'.format(hdr[hdr.rindex('opencv2/'):]) ) for decl in decls: name = decl[0] if name.startswith("struct") or name.startswith("class"): # class/struct p = name.find(" ") stype = name[:p] name = name[p+1:].strip() self.add_class(stype, name, decl) elif name.startswith("const"): # constant self.add_const(name.replace("const ", "").strip(), decl) else: # function self.add_func(decl) # step 1.5 check if all base classes exist for name, classinfo in self.classes.items(): if classinfo.base: chunks = classinfo.base.split('_') base = '_'.join(chunks) while base not in self.classes and len(chunks)>1: del chunks[-2] base = '_'.join(chunks) if base not in self.classes: print("Generator error: unable to resolve base %s for %s" % (classinfo.base, classinfo.name)) sys.exit(-1) classinfo.base = base classinfo.isalgorithm |= self.classes[base].isalgorithm self.classes[name] = classinfo # step 2: generate code for the classes and their methods classlist = list(self.classes.items()) classlist.sort() for name, classinfo in classlist: if classinfo.ismap: self.code_types.write(gen_template_map_type_cvt.substitute(name=name, cname=classinfo.cname)) else: if classinfo.issimple: templ = gen_template_simple_type_decl else: templ = gen_template_type_decl self.code_types.write(templ.substitute(name=name, wname=classinfo.wname, cname=classinfo.cname, cname1=("cv::Algorithm" if classinfo.isalgorithm else classinfo.cname))) # register classes in the same order as they have been declared. # this way, base classes will be registered in Python before their derivatives. classlist1 = [(classinfo.decl_idx, name, classinfo) for name, classinfo in classlist] classlist1.sort() for decl_idx, name, classinfo in classlist1: code = classinfo.gen_code(self.classes) self.code_types.write(code) if not classinfo.ismap: self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name,) ) # step 3: generate the code for all the global functions for ns_name, ns in sorted(self.namespaces.items()): if ns_name.split('.')[0] != 'cv': continue for name, func in sorted(ns.funcs.items()): code = func.gen_code(self.classes) self.code_funcs.write(code) self.gen_namespace(ns_name) self.gen_namespaces_reg() # step 4: generate the code for constants constlist = list(self.consts.items()) constlist.sort() for name, constinfo in constlist: self.gen_const_reg(constinfo) # That's it. Now save all the files self.save(output_path, "pyopencv_generated_include.h", self.code_include) self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs) self.save(output_path, "pyopencv_generated_types.h", self.code_types) self.save(output_path, "pyopencv_generated_type_reg.h", self.code_type_reg) self.save(output_path, "pyopencv_generated_ns_reg.h", self.code_ns_reg)
def process_module(module, path): hppparser = hp.CppHeaderParser() rstparser = rp.RstParser(hppparser) rstparser.parse(module, path) rst = rstparser.definitions hdrlist = [] for root, dirs, files in os.walk(os.path.join(path, "include")): for filename in fnmatch.filter(files, "*.h*"): hdrlist.append(os.path.join(root, filename)) if module == "gpu": hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_types.hpp")) hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu.hpp")) hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_stream_accessor.hpp")) decls = [] for hname in hdrlist: if not "ts_gtest.h" in hname: decls += hppparser.parse(hname, wmode=False) funcs = [] # not really needed to hardcode all the namespaces. Normally all they are collected automatically namespaces = ['cv', 'cv.gpu', 'cvflann', 'cvflann.anyimpl', 'cvflann.lsh', 'cv.flann', 'cv.linemod', 'cv.detail', 'cvtest', 'perf', 'cv.videostab'] classes = [] structs = [] # collect namespaces and classes/structs for decl in decls: if decl[0].startswith("const"): pass elif decl[0].startswith("class") or decl[0].startswith("struct"): if decl[0][0] == 'c': classes.append(decl) else: structs.append(decl) dotIdx = decl[0].rfind('.') if dotIdx > 0: namespace = decl[0][decl[0].find(' ')+1:dotIdx] if not [c for c in classes if c[0].endswith(namespace)] and not [s for s in structs if s[0].endswith(namespace)]: if namespace not in namespaces: namespaces.append(namespace) else: funcs.append(decl) clsnamespaces = [] # process classes for cl in classes: name = cl[0][cl[0].find(' ')+1:] if name.find('.') < 0 and not name.startswith("Cv"): logerror(ERROR_004_MISSEDNAMESPACE, "class " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention") clsnamespaces.append(name) if do_python_crosscheck and not name.startswith("cv.") and name.startswith("Cv"): clsnamespaces.append("cv." + name[2:]) if name.startswith("cv."): name = name[3:] name = name.replace(".", "::") sns = synonims.get(name, []) sns.append(name) for name in sns: doc = rst.get(name) if not doc: #TODO: class is not documented continue doc[DOCUMENTED_MARKER] = True # verify class marker if not doc.get("isclass"): logerror(ERROR_001_NOTACLASS, "class " + name + " is not marked as \"class\" in documentation", doc) else: # verify base signature = doc.get("class", "") signature = signature.replace(" public ", " ") namespaceIdx = signature.rfind("::") signature = ("class " + signature).strip() hdrsignature = ("class " + name + " " + cl[1]).replace(".", "::").replace("cv::","").strip() if signature != hdrsignature: logerror(ERROR_003_INCORRECTBASE, "invalid base class documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc) # process structs for st in structs: name = st[0][st[0].find(' ')+1:] if name.find('.') < 0 and not name.startswith("Cv"): logerror(ERROR_004_MISSEDNAMESPACE, "struct " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention") clsnamespaces.append(name) if name.startswith("cv."): name = name[3:] name = name.replace(".", "::") doc = rst.get(name) if not doc: #TODO: struct is not documented continue doc[DOCUMENTED_MARKER] = True # verify struct marker if not doc.get("isstruct"): logerror(ERROR_002_NOTASTRUCT, "struct " + name + " is not marked as \"struct\" in documentation", doc) else: # verify base signature = doc.get("class", "") signature = signature.replace(", public ", " ").replace(" public ", " ") signature = signature.replace(", protected ", " ").replace(" protected ", " ") signature = signature.replace(", private ", " ").replace(" private ", " ") signature = ("struct " + signature).strip() hdrsignature = (st[0] + " " + st[1]).replace("struct cv.", "struct ").replace(".", "::").strip() if signature != hdrsignature: logerror(ERROR_003_INCORRECTBASE, "invalid base struct documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc) print st, doc # process functions and methods flookup = {} for fn in funcs: name = fn[0] parent = None namespace = None for cl in clsnamespaces: if name.startswith(cl + "."): if cl.startswith(parent or ""): parent = cl if parent: name = name[len(parent) + 1:] for nm in namespaces: if parent.startswith(nm + "."): if nm.startswith(namespace or ""): namespace = nm if namespace: parent = parent[len(namespace) + 1:] else: for nm in namespaces: if name.startswith(nm + "."): if nm.startswith(namespace or ""): namespace = nm if namespace: name = name[len(namespace) + 1:] #print namespace, parent, name, fn[0] if not namespace and not parent and not name.startswith("cv") and not name.startswith("icv") and not name.startswith("CV_"): logerror(ERROR_004_MISSEDNAMESPACE, "function " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention") else: fdescr = (namespace, parent, name, fn) flookup_entry = flookup.get(fn[0], []) flookup_entry.append(fdescr) flookup[fn[0]] = flookup_entry if do_python_crosscheck: pyclsnamespaces = ["cv." + x[3:].replace(".", "_") for x in clsnamespaces] for name, doc in rst.iteritems(): decls = doc.get("decls") if not decls: continue for signature in decls: if signature[0] == "Python1": pname = signature[1][:signature[1].find('(')] try: fn = getattr(cv2.cv, pname[3:]) docstr = "cv." + fn.__doc__ except AttributeError: logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function: cv2." + pname, doc) continue docstring = docstr sign = signature[1] signature.append(DOCUMENTED_MARKER) # convert old signature to pydoc style if docstring.endswith("*"): docstring = docstring[:-1] s = None while s != sign: s = sign sign = re.sub(r"^(.*\(.*)\(.*?\)(.*\) *->)", "\\1_\\2", sign) s = None while s != sign: s = sign sign = re.sub(r"\s*,\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", " [, \\1\\2])", sign) sign = re.sub(r"\(\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", "([\\1\\2])", sign) sign = re.sub(r"\)\s*->\s*", ") -> ", sign) sign = sign.replace("-> convexHull", "-> CvSeq") sign = sign.replace("-> lines", "-> CvSeq") sign = sign.replace("-> boundingRects", "-> CvSeq") sign = sign.replace("-> contours", "-> CvSeq") sign = sign.replace("-> retval", "-> int") sign = sign.replace("-> detectedObjects", "-> CvSeqOfCvAvgComp") def retvalRplace(match): m = match.group(1) m = m.replace("CvScalar", "scalar") m = m.replace("CvMemStorage", "memstorage") m = m.replace("ROIplImage", "image") m = m.replace("IplImage", "image") m = m.replace("ROCvMat", "mat") m = m.replace("CvMat", "mat") m = m.replace("double", "float") m = m.replace("CvSubdiv2DPoint", "point") m = m.replace("CvBox2D", "Box2D") m = m.replace("IplConvKernel", "kernel") m = m.replace("CvHistogram", "hist") m = m.replace("CvSize", "width,height") m = m.replace("cvmatnd", "matND") m = m.replace("CvSeqOfCvConvexityDefect", "convexityDefects") mm = m.split(',') if len(mm) > 1: return "(" + ", ".join(mm) + ")" else: return m docstring = re.sub(r"(?<=-> )(.*)$", retvalRplace, docstring) docstring = docstring.replace("( [, ", "([") if sign != docstring: logerror(ERROR_006_INVALIDPYOLDDOC, "old-style documentation differs from pydoc\npydoc: " + docstring + "\nfixup: " + sign + "\ncvdoc: " + signature[1], doc) elif signature[0] == "Python2": pname = signature[1][4:signature[1].find('(')] cvname = "cv." + pname parent = None for cl in pyclsnamespaces: if cvname.startswith(cl + "."): if cl.startswith(parent or ""): parent = cl try: if parent: instance, clsname = get_cv2_object(parent) fn = getattr(instance, cvname[len(parent)+1:]) docstr = fn.__doc__ docprefix = "cv2." + clsname + "." else: fn = getattr(cv2, pname) docstr = fn.__doc__ docprefix = "cv2." except AttributeError: if parent: logerror(ERROR_005_MISSINGPYFUNC, "could not load documented member of " + parent + " class: cv2." + pname, doc) else: logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function cv2." + pname, doc) signature.append(DOCUMENTED_MARKER) # stop subsequent errors continue docstrings = [docprefix + s.replace("([, ", "([") for s in docstr.split(" or ")] if not signature[1] in docstrings: pydocs = "\npydoc: ".join(docstrings) logerror(ERROR_007_INVALIDPYDOC, "documentation differs from pydoc\npydoc: " + pydocs + "\ncvdoc: " + signature[1], doc) signature.append(DOCUMENTED_MARKER) # verify C/C++ signatures for name, doc in rst.iteritems(): decls = doc.get("decls") if not decls: continue for signature in decls: if signature[0] == "C" or signature[0] == "C++": if "template" in (signature[2][1] or ""): # TODO find a way to validate templates signature.append(DOCUMENTED_MARKER) continue fd = flookup.get(signature[2][0]) if not fd: if signature[2][0].startswith("cv."): fd = flookup.get(signature[2][0][3:]) if not fd: continue else: signature[2][0] = signature[2][0][3:] if signature[0] == "C": ffd = [f for f in fd if not f[0] and not f[1]] # filter out C++ stuff if not ffd: if fd[0][1]: logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually member of " + fd[0][1] + " class", doc) elif fd[0][0]: logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually placed in " + fd[0][0] + " namespace", doc) fd = ffd error = None for f in fd: match, error = compareSignatures(signature[2], f[3]) if match: signature.append(DOCUMENTED_MARKER) break if signature[-1] != DOCUMENTED_MARKER: candidates = "\n\t".join([formatSignature(f[3]) for f in fd]) logerror(ERROR_009_OVERLOADNOTFOUND, signature[0] + " function " + signature[2][0].replace(".","::") + " is documented but misses in headers (" + error + ").\nDocumented as:\n\t" + signature[1] + "\nCandidates are:\n\t" + candidates, doc) signature.append(DOCUMENTED_MARKER) # to stop subsequent error on this function # verify that all signatures was found in the library headers for name, doc in rst.iteritems(): # if doc.get(DOCUMENTED_MARKER, False): # continue # this class/struct was found if not doc.get(DOCUMENTED_MARKER, False) and (doc.get("isclass", False) or doc.get("isstruct", False)): if name in doc_signatures_whitelist: continue logerror(ERROR_010_UNKNOWNCLASS, "class/struct " + name + " is mentioned in documentation but is not found in OpenCV headers", doc) for d in doc.get("decls", []): if d[-1] != DOCUMENTED_MARKER: if d[0] == "C" or d[0] =="C++" or (do_python_crosscheck and d[0].startswith("Python")): if d[0][0] == 'C': sname = d[2][0][3:].replace(".", "::") if sname in defines: #TODO: need to find a way to verify #define's continue else: sname = d[1][:d[1].find("(")] prefixes = [x for x in doc_signatures_whitelist if sname.startswith(x)] if prefixes: # TODO: member of template class continue logerror(ERROR_011_UNKNOWNFUNC, d[0] + " function " + sname + " is documented but is not found in OpenCV headers. It is documented as:\n\t" + d[1], doc)
def gen(self, srcfiles, output_path): self.clear() self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True, generate_gpumat_decls=True) # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = self.parser.parse(hdr) if len(decls) == 0: continue if hdr.find('opencv2/') >= 0: #Avoid including the shadow files self.code_include.write('#include "{0}"\n'.format( hdr[hdr.rindex('opencv2/'):])) for decl in decls: name = decl[0] if name.startswith("struct") or name.startswith("class"): # class/struct p = name.find(" ") stype = name[:p] name = name[p + 1:].strip() self.add_class(stype, name, decl) elif name.startswith("const"): # constant self.add_const(name.replace("const ", "").strip(), decl) elif name.startswith("enum"): # enum self.add_enum(name.rsplit(" ", 1)[1], decl) else: # function self.add_func(decl) # step 1.5 check if all base classes exist for name, classinfo in self.classes.items(): if classinfo.base: chunks = classinfo.base.split('_') base = '_'.join(chunks) while base not in self.classes and len(chunks) > 1: del chunks[-2] base = '_'.join(chunks) if base not in self.classes: print("Generator error: unable to resolve base %s for %s" % (classinfo.base, classinfo.name)) sys.exit(-1) base_instance = self.classes[base] classinfo.base = base classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag: # doesn't work for trees(graphs) with depth > 2 self.classes[name] = classinfo # tree-based propagation of 'isalgorithm' processed = dict() def process_isalgorithm(classinfo): if classinfo.isalgorithm or classinfo in processed: return classinfo.isalgorithm res = False if classinfo.base: res = process_isalgorithm(self.classes[classinfo.base]) #assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base classinfo.isalgorithm |= res res = classinfo.isalgorithm processed[classinfo] = True return res for name, classinfo in self.classes.items(): process_isalgorithm(classinfo) # step 2: generate code for the classes and their methods classlist = list(self.classes.items()) classlist.sort() for name, classinfo in classlist: self.code_types.write("//{}\n".format(80 * "=")) self.code_types.write("// {} ({})\n".format( name, 'Map' if classinfo.ismap else 'Generic')) self.code_types.write("//{}\n".format(80 * "=")) self.code_types.write(classinfo.gen_code(self)) if classinfo.ismap: self.code_types.write( gen_template_map_type_cvt.substitute( name=classinfo.name, cname=classinfo.cname)) else: mappable_code = "\n".join([ gen_template_mappable.substitute(cname=classinfo.cname, mappable=mappable) for mappable in classinfo.mappables ]) code = gen_template_type_decl.substitute( name=classinfo.name, cname=classinfo.cname if classinfo.issimple else "Ptr<{}>".format(classinfo.cname), mappable_code=mappable_code) self.code_types.write(code) # register classes in the same order as they have been declared. # this way, base classes will be registered in Python before their derivatives. classlist1 = [(classinfo.decl_idx, name, classinfo) for name, classinfo in classlist] classlist1.sort() for decl_idx, name, classinfo in classlist1: if classinfo.ismap: continue self.code_type_publish.write(classinfo.gen_def(self)) # step 3: generate the code for all the global functions for ns_name, ns in sorted(self.namespaces.items()): if ns_name.split('.')[0] != 'cv': continue for name, func in sorted(ns.funcs.items()): if func.isconstructor: continue code = func.gen_code(self) self.code_funcs.write(code) self.gen_namespace(ns_name) self.code_ns_init.write('CVPY_MODULE("{}", {});\n'.format( ns_name[2:], normalize_class_name(ns_name))) # step 4: generate the code for enum types enumlist = list(self.enums.values()) enumlist.sort() for name in enumlist: self.gen_enum_reg(name) # step 5: generate the code for constants constlist = list(self.consts.items()) constlist.sort() for name, constinfo in constlist: self.gen_const_reg(constinfo) # That's it. Now save all the files self.save(output_path, "pyopencv_generated_include.h", self.code_include) self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs) self.save(output_path, "pyopencv_generated_enums.h", self.code_enums) self.save(output_path, "pyopencv_generated_types.h", self.code_type_publish) self.save(output_path, "pyopencv_generated_types_content.h", self.code_types) self.save(output_path, "pyopencv_generated_modules.h", self.code_ns_init) self.save(output_path, "pyopencv_generated_modules_content.h", self.code_ns_reg) self.save_json(output_path, "pyopencv_signatures.json", self.py_signatures)
import rst_parser parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", help="Print verbose log to stdout", action="store_true", default=False) parser.add_option("", "--no-warnings", dest="warnings", help="Hide warning messages", action="store_false", default=True) parser.add_option("", "--no-errors", dest="errors", help="Hide error messages", action="store_false", default=True) parser.add_option("", "--modules", dest="modules", help="comma-separated list of modules to generate comments", metavar="MODS", default=",".join(rst_parser.allmodules)) (options, args) = parser.parse_args(sys.argv) options.modules = options.modules.split(",") if len(args) < 2 or len(options.modules) < 1: parser.print_help() exit(0) parser = rst_parser.RstParser(hdr_parser.CppHeaderParser()) for m in options.modules: parser.parse(m, os.path.join(selfpath, "../../" + m)) parser.printSummary() generator = JavadocGenerator(parser.definitions, options.modules) generator.verbose = options.verbose generator.show_warnings = options.warnings generator.show_errors = options.errors for path in args: folder = os.path.abspath(path) for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]: outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java")) generator.document(jfile, outfile)
def gen(self, srcfiles, output_path): self.clear() parser = hdr_parser.CppHeaderParser() # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = parser.parse(hdr) for decl in decls: name = decl[0] if name.startswith("struct") or name.startswith("class"): # class/struct p = name.find(" ") stype = name[:p] name = name[p + 1:].strip() self.add_class(stype, name, decl) elif name.startswith("const"): # constant self.add_const(name.replace("const ", "").strip(), decl) else: # function self.add_func(decl) # step 2: generate code for the classes and their methods classlist = self.classes.items() classlist.sort() for name, classinfo in classlist: if classinfo.ismap: self.code_types.write( gen_template_map_type_cvt.substitute( name=name, cname=classinfo.cname)) else: if classinfo.issimple: templ = gen_template_simple_type_decl else: templ = gen_template_type_decl self.code_types.write( templ.substitute( name=name, wname=classinfo.wname, cname=classinfo.cname, cname1=("cv::Algorithm" if classinfo.isalgorithm else classinfo.cname))) # register classes in the same order as they have been declared. # this way, base classes will be registered in Python before their derivatives. classlist1 = [(classinfo.decl_idx, name, classinfo) for name, classinfo in classlist] classlist1.sort() for decl_idx, name, classinfo in classlist1: code = classinfo.gen_code(self.classes) self.code_types.write(code) if not classinfo.ismap: self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name, )) # step 3: generate the code for all the global functions funclist = self.funcs.items() funclist.sort() for name, func in funclist: code = func.gen_code(self.classes) self.code_funcs.write(code) self.code_func_tab.write(func.get_tab_entry()) # step 4: generate the code for constants constlist = self.consts.items() constlist.sort() for name, constinfo in constlist: self.gen_const_reg(constinfo) # That's it. Now save all the files self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs) self.save(output_path, "pyopencv_generated_func_tab.h", self.code_func_tab) self.save(output_path, "pyopencv_generated_const_reg.h", self.code_const_reg) self.save(output_path, "pyopencv_generated_types.h", self.code_types) self.save(output_path, "pyopencv_generated_type_reg.h", self.code_type_reg)
def gen_tree(srcfiles): parser = hdr_parser.CppHeaderParser(generate_umat_decls=False, generate_gpumat_decls=False) allowed_func_list = [] with open("funclist.csv", "r") as f: allowed_func_list = f.readlines() allowed_func_list = [x[:-1] for x in allowed_func_list] count = 0 # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = parser.parse(hdr) for ns in parser.namespaces: ns = ns.replace('.', '::') if ns not in namespaces: namespaces[ns] = NameSpaceInfo(ns) count += len(decls) if len(decls) == 0: continue if hdr.find('opencv2/') >= 0: #Avoid including the shadow files # code_include.write( '#include "{0}"\n'.format(hdr[hdr.rindex('opencv2/'):]) ) pass for decl in decls: name = decl[0] if name.startswith("struct") or name.startswith("class"): # class/struct p = name.find(" ") stype = name[:p] name = name[p + 1:].strip() add_class(stype, name, decl) elif name.startswith("const"): # constant assert (0) add_const(name.replace("const ", "").strip(), decl) elif name.startswith("enum"): # enum add_enum(name.rsplit(" ", 1)[1], decl) else: # function if decl[0] in allowed_func_list: add_func(decl) # step 1.5 check if all base classes exist # print(classes) for name, classinfo in classes.items(): if classinfo.base: base = classinfo.base # print(base) if base not in classes: print("Generator error: unable to resolve base %s for %s" % (classinfo.base, classinfo.name)) sys.exit(-1) base_instance = classes[base] classinfo.base = base classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag: # doesn't work for trees(graphs) with depth > 2 classes[name] = classinfo # tree-based propagation of 'isalgorithm' processed = dict() def process_isalgorithm(classinfo): if classinfo.isalgorithm or classinfo in processed: return classinfo.isalgorithm res = False if classinfo.base: res = process_isalgorithm(classes[classinfo.base]) #assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base classinfo.isalgorithm |= res res = classinfo.isalgorithm processed[classinfo] = True return res for name, classinfo in classes.items(): process_isalgorithm(classinfo) for name, ns in namespaces.items(): if name.split('.')[-1] == '': continue ns.registered = [] for name, cl in ns.classes.items(): registered_types.append(get_template_arg(name)) ns.registered.append(cl.mapped_name) nss, clss, bs = split_decl_name(name) type_paths[bs] = [name.replace("::", ".")] type_paths["::".join(clss + [bs])] = [name.replace("::", ".")] for e1, e2 in ns.enums.items(): registered_types.append(get_template_arg(e2[0])) registered_types.append( get_template_arg(e2[0]).replace('::', '_')) #whyyy typedef ns.registered.append(e2[1]) ns.register_types = list(set(ns.register_types)) ns.register_types = [ tp for tp in ns.register_types if not registered_tp_search(tp) and not tp in ns.registered ] for tp in ns.register_types: registered_types.append(get_template_arg(tp)) ns.registered.append(get_template_arg(tp)) default_valuesr = list(set(default_values)) # registered_types = registered_types + ns.register_types return namespaces, default_valuesr
def gen(self, module_roots, modules, extras, output_dir): """ Generate a set of Matlab mex source files by parsing exported symbols in a set of C++ headers. The headers can be input in one (or both) of two methods: 1. specify module_root and modules Given a path to the OpenCV module root and a list of module names, the headers to parse are implicitly constructed. 2. specifiy header locations explicitly in extras Each element in the list of extras must be of the form: 'namespace=/full/path/to/extra/header.hpp' where 'namespace' is the namespace in which the definitions should be added. The output_dir specifies the directory to write the generated sources to. """ # dynamically import the parsers from jinja2 import Environment, FileSystemLoader import hdr_parser import rst_parser # parse each of the files and store in a dictionary # as a separate "namespace" parser = hdr_parser.CppHeaderParser() rst = rst_parser.RstParser(parser) rst_parser.verbose = False rst_parser.show_warnings = False rst_parser.show_errors = False rst_parser.show_critical_errors = False ns = dict((key, []) for key in modules) doc = dict((key, []) for key in modules) path_template = Template('${module}/include/opencv2/${module}.hpp') for module in modules: for module_root in module_roots: # construct a header path from the module root and a path template header = os.path.join(module_root, path_template.substitute(module=module)) if os.path.isfile(header): break else: raise Exception('no header found for module %s!' % module) # parse the definitions ns[module] = parser.parse(header) # parse the documentation rst.parse(module, os.path.join(module_root, module)) doc[module] = rst.definitions rst.definitions = {} for extra in extras: module = extra.split("=")[0] header = extra.split("=")[1] ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header) # cleanify the parser output parse_tree = ParseTree() parse_tree.build(ns) # setup the template engine template_dir = os.path.join(os.path.dirname(__file__), 'templates') jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True) # add the custom filters jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant jtemplate.filters['convertibleToInt'] = convertibleToInt jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase jtemplate.filters['toUnderCase'] = toUnderCase jtemplate.filters['matlabURL'] = matlabURL jtemplate.filters['stripTags'] = stripTags jtemplate.filters['filename'] = filename jtemplate.filters['comment'] = comment jtemplate.filters['inputs'] = inputs jtemplate.filters['ninputs'] = ninputs jtemplate.filters['outputs'] = outputs jtemplate.filters['noutputs'] = noutputs jtemplate.filters['qualify'] = qualify jtemplate.filters['slugify'] = slugify jtemplate.filters['only'] = only jtemplate.filters['void'] = void jtemplate.filters['not'] = flip # load the templates tfunction = jtemplate.get_template('template_function_base.cpp') tclassm = jtemplate.get_template('template_class_base.m') tclassc = jtemplate.get_template('template_class_base.cpp') tdoc = jtemplate.get_template('template_doc_base.m') tconst = jtemplate.get_template('template_map_base.m') # create the build directory output_source_dir = output_dir+'/src' output_private_dir = output_source_dir+'/private' output_class_dir = output_dir+'/+cv' output_map_dir = output_dir+'/map' if not os.path.isdir(output_source_dir): os.makedirs(output_source_dir) if not os.path.isdir(output_private_dir): os.makedirs(output_private_dir) if not os.path.isdir(output_class_dir): os.makedirs(output_class_dir) if not os.path.isdir(output_map_dir): os.makedirs(output_map_dir) # populate templates for namespace in parse_tree.namespaces: # functions for method in namespace.methods: populated = tfunction.render(fun=method, time=time, includes=namespace.name) with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f: f.write(populated.encode('utf-8')) if namespace.name in doc and method.name in doc[namespace.name]: populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time) with open(output_class_dir+'/'+method.name+'.m', 'wb') as f: f.write(populated.encode('utf-8')) # classes for clss in namespace.classes: # cpp converter populated = tclassc.render(clss=clss, time=time) with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f: f.write(populated.encode('utf-8')) # matlab classdef populated = tclassm.render(clss=clss, time=time) with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f: f.write(populated.encode('utf-8')) # create a global constants lookup table const = dict(constants(todict(parse_tree.namespaces))) populated = tconst.render(constants=const, time=time) with open(output_dir+'/cv.m', 'wb') as f: f.write(populated.encode('utf-8'))
if len(sys.argv) > 2: system_include_dir_search.insert(0, sys.argv[2]) system_include_dir = None for include_dir in system_include_dir_search: if os.path.exists(os.path.join(include_dir, src_files[0])): system_include_dir = include_dir break print('Using include dir: {}'.format(system_include_dir)) # TODO enable UMat support, and make a wrapper for handling both Mat and UMat identically generate_umat = False parser = hdr_parser.CppHeaderParser(generate_umat_decls=generate_umat, generate_gpumat_decls=generate_umat) functions = [] enums = [] enum_map = {} defined_enum_consts = set() classes = {} overload_counts = {} ocaml_overload_counts = {} draw_functions = [] def add_struct(struct): type_manager.add_type( type_manager.CustomType( struct.cpp_name, struct.cpp_name, 'unit ptr',