def __init__(self):
self.bindings = []
self.wrapper_funcs = []
self.classes = {}
self.namespaces = {}
self.enums = {}
self.parser = hdr_parser.CppHeaderParser()
self.class_idx = 0
def check_module_docs(self, name):
self.parser = hp.CppHeaderParser()
decls = []
self.fmap = {}
for hname in opencv_hdr_list:
if hname.startswith("../modules/" + name):
decls += self.parser.parse(hname, wmode=False)
for d in decls:
fname = d[0]
if not fname.startswith("struct") and not fname.startswith(
"class") and not fname.startswith("const"):
dlist = self.fmap.get(fname, [])
dlist.append(d)
self.fmap[fname] = dlist
self.missing_docfunc_list = []
doclist = glob.glob("../modules/" + name + "/doc/*.rst")
for d in doclist:
self.process_rst(d)
print "\n\n########## The list of undocumented functions: ###########\n\n"
misscount = 0
fkeys = sorted(self.fmap.keys())
for f in fkeys:
# skip undocumented destructors
if "~" in f:
continue
decls = self.fmap[f]
fcomps = f.split(".")
prefix = ""
wlist_decls = []
for c in fcomps:
prefix = (prefix + "." + c).lstrip(".")
wlist_decls = self.whitelist.get(prefix, [])
if wlist_decls == "*":
break
if wlist_decls == "*":
continue
wlist_decls = [self.decl2str(d) for d in wlist_decls]
for d in decls:
dstr = self.decl2str(d)
# special hack for ML: skip old variants of the methods
if name == "ml" and ("CvMat" in dstr):
continue
if dstr not in wlist_decls:
misscount += 1
print "%s %s(%s)" % (d[1], d[0].replace(
".", "::"), ", ".join(
[a[0] + " " + a[1] for a in d[3]]))
print "\n\n\nundocumented functions in %s: %d" % (name, misscount)
def get_typelist(headers):
group_map = {}
import hdr_parser
headers = list(headers) + hdr_parser.opencv_hdr_list
headers = map(os.path.abspath, headers)
headers = set(headers)
parser = hdr_parser.CppHeaderParser(False)
types = {}
namespaces = {}
groups = {}
for header in sorted(headers):
header = header.replace("\\", "/")
print("INFO Processing header file:", header)
if not header or not os.path.exists(header):
continue
group = "modules/" + re.match(r".*/modules/(.+)", header).group(1)
# group = group_map.get(group, None)
for declaration in parser.parse(header):
function = TypeDef.from_declaration(declaration)
if not function:
continue
if function.type == "func":
namespaces[function.name] = function.namespace
groups[function.name] = group
for arg in function.args:
name = "{function.name}:{arg.name}".format(**locals()).lower()
type = arg.type
if type:
if types.get(name, type) != type:
print("ERROR Inconsistent types: " + name)
types[name] = type
return types, namespaces, groups
def read_whitelist(self):
self.whitelist = {}
try:
wf = open("check_docs_whitelist.txt", "rt")
except IOError:
return
self.parser = hp.CppHeaderParser()
for l in wf.readlines():
cpos = l.find("#")
if cpos >= 0:
l = l[:cpos]
l = l.strip()
if not l:
continue
rst_decl = None
if "(" in l:
l = l.replace("cv::", "")
rst_decl = self.parser.parse_func_decl_no_wrap(l)
fname = rst_decl[0]
else:
fname = l.replace("::", ".")
complist = fname.split(".")
prefix = ""
alreadyListed = False
wl = []
for c in complist:
prefix = (prefix + "." + c).lstrip(".")
wl = self.whitelist.get(prefix, [])
if wl == "*":
break
if wl == "*":
continue
if not rst_decl:
self.whitelist[fname] = "*"
else:
wl.append(rst_decl)
self.whitelist[fname] = wl
wf.close()
def gen(self, srcfiles, output_path):
self.clear()
self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True)
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = self.parser.parse(hdr)
if len(decls) == 0:
continue
self.code_include.write('#include "{0}"\n'.format(
hdr[hdr.rindex('src/'):]))
for decl in decls:
name = decl[0]
if name.startswith("struct") or name.startswith("class"):
# class/struct
p = name.find(" ")
stype = name[:p]
name = name[p + 1:].strip()
self.add_class(stype, name, decl)
elif name.startswith("const"):
# constant
self.add_const(name.replace("const ", "").strip(), decl)
else:
# function
self.add_func(decl)
# step 1.5 check if all base classes exist
for name, classinfo in self.classes.items():
if classinfo.base:
chunks = classinfo.base.split('_')
base = '_'.join(chunks)
while base not in self.classes and len(chunks) > 1:
del chunks[-2]
base = '_'.join(chunks)
if base not in self.classes:
print("Generator error: unable to resolve base %s for %s" %
(classinfo.base, classinfo.name))
sys.exit(-1)
base_instance = self.classes[base]
classinfo.base = base
classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag:
# doesn't work for trees(graphs) with depth > 2
self.classes[name] = classinfo
# tree-based propagation of 'isalgorithm'
processed = dict()
def process_isalgorithm(classinfo):
if classinfo.isalgorithm or classinfo in processed:
return classinfo.isalgorithm
res = False
if classinfo.base:
res = process_isalgorithm(self.classes[classinfo.base])
#assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base
classinfo.isalgorithm |= res
res = classinfo.isalgorithm
processed[classinfo] = True
return res
for name, classinfo in self.classes.items():
process_isalgorithm(classinfo)
# step 2: generate code for the classes and their methods
classlist = list(self.classes.items())
classlist.sort()
for name, classinfo in classlist:
if classinfo.ismap:
self.code_types.write(
gen_template_map_type_cvt.substitute(
modulePrefix=modulePrefix,
name=name,
cname=classinfo.cname))
else:
if classinfo.issimple:
templ = gen_template_simple_type_decl
else:
templ = gen_template_type_decl
self.code_types.write(
templ.substitute(
modulePrefix=modulePrefix,
name=name,
wname=classinfo.wname,
cname=classinfo.cname,
sname=classinfo.sname,
cname1=("cv::Algorithm" if classinfo.isalgorithm else
classinfo.cname)))
# register classes in the same order as they have been declared.
# this way, base classes will be registered in Python before their derivatives.
classlist1 = [(classinfo.decl_idx, name, classinfo)
for name, classinfo in classlist]
classlist1.sort()
for decl_idx, name, classinfo in classlist1:
code = classinfo.gen_code(self)
self.code_types.write(code)
if not classinfo.ismap:
self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name, ))
self.code_type_publish.write(
"PUBLISH_OBJECT(\"{name}\", {modulePrefix}_{name}_Type);\n"
.format(name=classinfo.name, modulePrefix=modulePrefix))
# step 3: generate the code for all the global functions
for ns_name, ns in sorted(self.namespaces.items()):
#Chandra disabled
#if ns_name.split('.')[0] != 'cv':
# continue
for name, func in sorted(ns.funcs.items()):
if func.isconstructor:
continue
code = func.gen_code(self)
self.code_funcs.write(code)
self.gen_namespace(ns_name)
self.gen_namespaces_reg()
# step 4: generate the code for constants
constlist = list(self.consts.items())
constlist.sort()
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# That's it. Now save all the files
self.save(output_path, modulePrefix + "_generated_include.h",
self.code_include)
self.save(output_path, modulePrefix + "_generated_funcs.h",
self.code_funcs)
self.save(output_path, modulePrefix + "_generated_types.h",
self.code_types)
self.save(output_path, modulePrefix + "_generated_type_reg.h",
self.code_type_reg)
self.save(output_path, modulePrefix + "_generated_ns_reg.h",
self.code_ns_reg)
self.save(output_path, modulePrefix + "_generated_type_publish.h",
self.code_type_publish)
self.save_json(output_path, modulePrefix + "_signatures.json",
self.py_signatures)
if len(sys.argv) < 2:
print "Usage:\n", os.path.basename(sys.argv[0]), " <module path>"
exit(0)
if len(sys.argv) >= 3:
if sys.argv[2].lower() == "verbose":
verbose = True
rst_parser_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
hdr_parser_path = os.path.join(rst_parser_dir, "../../python/src2")
sys.path.append(hdr_parser_path)
import hdr_parser
module = sys.argv[1]
if module != "all" and not os.path.isdir(os.path.join(rst_parser_dir, "../../" + module)):
print "RST parser error E%03d: module \"%s\" could not be found." % (ERROR_010_NOMODULE, module)
exit(1)
parser = RstParser(hdr_parser.CppHeaderParser())
if module == "all":
for m in allmodules:
parser.parse(m, os.path.join(rst_parser_dir, "../../" + m))
else:
parser.parse(module, os.path.join(rst_parser_dir, "../../" + module))
# summary
parser.printSummary()
def gen(self, srcfiles, output_path):
self.clear()
self.parser = hdr_parser.CppHeaderParser()
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = self.parser.parse(hdr)
if len(decls) == 0:
continue
self.code_include.write( '#include "{0}"\n'.format(hdr[hdr.rindex('opencv2/'):]) )
for decl in decls:
name = decl[0]
if name.startswith("struct") or name.startswith("class"):
# class/struct
p = name.find(" ")
stype = name[:p]
name = name[p+1:].strip()
self.add_class(stype, name, decl)
elif name.startswith("const"):
# constant
self.add_const(name.replace("const ", "").strip(), decl)
else:
# function
self.add_func(decl)
# step 1.5 check if all base classes exist
for name, classinfo in self.classes.items():
if classinfo.base:
chunks = classinfo.base.split('_')
base = '_'.join(chunks)
while base not in self.classes and len(chunks)>1:
del chunks[-2]
base = '_'.join(chunks)
if base not in self.classes:
print("Generator error: unable to resolve base %s for %s"
% (classinfo.base, classinfo.name))
sys.exit(-1)
classinfo.base = base
classinfo.isalgorithm |= self.classes[base].isalgorithm
self.classes[name] = classinfo
# step 2: generate code for the classes and their methods
classlist = list(self.classes.items())
classlist.sort()
for name, classinfo in classlist:
if classinfo.ismap:
self.code_types.write(gen_template_map_type_cvt.substitute(name=name, cname=classinfo.cname))
else:
if classinfo.issimple:
templ = gen_template_simple_type_decl
else:
templ = gen_template_type_decl
self.code_types.write(templ.substitute(name=name, wname=classinfo.wname, cname=classinfo.cname,
cname1=("cv::Algorithm" if classinfo.isalgorithm else classinfo.cname)))
# register classes in the same order as they have been declared.
# this way, base classes will be registered in Python before their derivatives.
classlist1 = [(classinfo.decl_idx, name, classinfo) for name, classinfo in classlist]
classlist1.sort()
for decl_idx, name, classinfo in classlist1:
code = classinfo.gen_code(self.classes)
self.code_types.write(code)
if not classinfo.ismap:
self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name,) )
# step 3: generate the code for all the global functions
for ns_name, ns in sorted(self.namespaces.items()):
if ns_name.split('.')[0] != 'cv':
continue
for name, func in sorted(ns.funcs.items()):
code = func.gen_code(self.classes)
self.code_funcs.write(code)
self.gen_namespace(ns_name)
self.gen_namespaces_reg()
# step 4: generate the code for constants
constlist = list(self.consts.items())
constlist.sort()
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# That's it. Now save all the files
self.save(output_path, "pyopencv_generated_include.h", self.code_include)
self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs)
self.save(output_path, "pyopencv_generated_types.h", self.code_types)
self.save(output_path, "pyopencv_generated_type_reg.h", self.code_type_reg)
self.save(output_path, "pyopencv_generated_ns_reg.h", self.code_ns_reg)
def process_module(module, path):
hppparser = hp.CppHeaderParser()
rstparser = rp.RstParser(hppparser)
rstparser.parse(module, path)
rst = rstparser.definitions
hdrlist = []
for root, dirs, files in os.walk(os.path.join(path, "include")):
for filename in fnmatch.filter(files, "*.h*"):
hdrlist.append(os.path.join(root, filename))
if module == "gpu":
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_types.hpp"))
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu.hpp"))
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_stream_accessor.hpp"))
decls = []
for hname in hdrlist:
if not "ts_gtest.h" in hname:
decls += hppparser.parse(hname, wmode=False)
funcs = []
# not really needed to hardcode all the namespaces. Normally all they are collected automatically
namespaces = ['cv', 'cv.gpu', 'cvflann', 'cvflann.anyimpl', 'cvflann.lsh', 'cv.flann', 'cv.linemod', 'cv.detail', 'cvtest', 'perf', 'cv.videostab']
classes = []
structs = []
# collect namespaces and classes/structs
for decl in decls:
if decl[0].startswith("const"):
pass
elif decl[0].startswith("class") or decl[0].startswith("struct"):
if decl[0][0] == 'c':
classes.append(decl)
else:
structs.append(decl)
dotIdx = decl[0].rfind('.')
if dotIdx > 0:
namespace = decl[0][decl[0].find(' ')+1:dotIdx]
if not [c for c in classes if c[0].endswith(namespace)] and not [s for s in structs if s[0].endswith(namespace)]:
if namespace not in namespaces:
namespaces.append(namespace)
else:
funcs.append(decl)
clsnamespaces = []
# process classes
for cl in classes:
name = cl[0][cl[0].find(' ')+1:]
if name.find('.') < 0 and not name.startswith("Cv"):
logerror(ERROR_004_MISSEDNAMESPACE, "class " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
clsnamespaces.append(name)
if do_python_crosscheck and not name.startswith("cv.") and name.startswith("Cv"):
clsnamespaces.append("cv." + name[2:])
if name.startswith("cv."):
name = name[3:]
name = name.replace(".", "::")
sns = synonims.get(name, [])
sns.append(name)
for name in sns:
doc = rst.get(name)
if not doc:
#TODO: class is not documented
continue
doc[DOCUMENTED_MARKER] = True
# verify class marker
if not doc.get("isclass"):
logerror(ERROR_001_NOTACLASS, "class " + name + " is not marked as \"class\" in documentation", doc)
else:
# verify base
signature = doc.get("class", "")
signature = signature.replace(" public ", " ")
namespaceIdx = signature.rfind("::")
signature = ("class " + signature).strip()
hdrsignature = ("class " + name + " " + cl[1]).replace(".", "::").replace("cv::","").strip()
if signature != hdrsignature:
logerror(ERROR_003_INCORRECTBASE, "invalid base class documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
# process structs
for st in structs:
name = st[0][st[0].find(' ')+1:]
if name.find('.') < 0 and not name.startswith("Cv"):
logerror(ERROR_004_MISSEDNAMESPACE, "struct " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
clsnamespaces.append(name)
if name.startswith("cv."):
name = name[3:]
name = name.replace(".", "::")
doc = rst.get(name)
if not doc:
#TODO: struct is not documented
continue
doc[DOCUMENTED_MARKER] = True
# verify struct marker
if not doc.get("isstruct"):
logerror(ERROR_002_NOTASTRUCT, "struct " + name + " is not marked as \"struct\" in documentation", doc)
else:
# verify base
signature = doc.get("class", "")
signature = signature.replace(", public ", " ").replace(" public ", " ")
signature = signature.replace(", protected ", " ").replace(" protected ", " ")
signature = signature.replace(", private ", " ").replace(" private ", " ")
signature = ("struct " + signature).strip()
hdrsignature = (st[0] + " " + st[1]).replace("struct cv.", "struct ").replace(".", "::").strip()
if signature != hdrsignature:
logerror(ERROR_003_INCORRECTBASE, "invalid base struct documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
print st, doc
# process functions and methods
flookup = {}
for fn in funcs:
name = fn[0]
parent = None
namespace = None
for cl in clsnamespaces:
if name.startswith(cl + "."):
if cl.startswith(parent or ""):
parent = cl
if parent:
name = name[len(parent) + 1:]
for nm in namespaces:
if parent.startswith(nm + "."):
if nm.startswith(namespace or ""):
namespace = nm
if namespace:
parent = parent[len(namespace) + 1:]
else:
for nm in namespaces:
if name.startswith(nm + "."):
if nm.startswith(namespace or ""):
namespace = nm
if namespace:
name = name[len(namespace) + 1:]
#print namespace, parent, name, fn[0]
if not namespace and not parent and not name.startswith("cv") and not name.startswith("icv") and not name.startswith("CV_"):
logerror(ERROR_004_MISSEDNAMESPACE, "function " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
else:
fdescr = (namespace, parent, name, fn)
flookup_entry = flookup.get(fn[0], [])
flookup_entry.append(fdescr)
flookup[fn[0]] = flookup_entry
if do_python_crosscheck:
pyclsnamespaces = ["cv." + x[3:].replace(".", "_") for x in clsnamespaces]
for name, doc in rst.iteritems():
decls = doc.get("decls")
if not decls:
continue
for signature in decls:
if signature[0] == "Python1":
pname = signature[1][:signature[1].find('(')]
try:
fn = getattr(cv2.cv, pname[3:])
docstr = "cv." + fn.__doc__
except AttributeError:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function: cv2." + pname, doc)
continue
docstring = docstr
sign = signature[1]
signature.append(DOCUMENTED_MARKER)
# convert old signature to pydoc style
if docstring.endswith("*"):
docstring = docstring[:-1]
s = None
while s != sign:
s = sign
sign = re.sub(r"^(.*\(.*)\(.*?\)(.*\) *->)", "\\1_\\2", sign)
s = None
while s != sign:
s = sign
sign = re.sub(r"\s*,\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", " [, \\1\\2])", sign)
sign = re.sub(r"\(\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", "([\\1\\2])", sign)
sign = re.sub(r"\)\s*->\s*", ") -> ", sign)
sign = sign.replace("-> convexHull", "-> CvSeq")
sign = sign.replace("-> lines", "-> CvSeq")
sign = sign.replace("-> boundingRects", "-> CvSeq")
sign = sign.replace("-> contours", "-> CvSeq")
sign = sign.replace("-> retval", "-> int")
sign = sign.replace("-> detectedObjects", "-> CvSeqOfCvAvgComp")
def retvalRplace(match):
m = match.group(1)
m = m.replace("CvScalar", "scalar")
m = m.replace("CvMemStorage", "memstorage")
m = m.replace("ROIplImage", "image")
m = m.replace("IplImage", "image")
m = m.replace("ROCvMat", "mat")
m = m.replace("CvMat", "mat")
m = m.replace("double", "float")
m = m.replace("CvSubdiv2DPoint", "point")
m = m.replace("CvBox2D", "Box2D")
m = m.replace("IplConvKernel", "kernel")
m = m.replace("CvHistogram", "hist")
m = m.replace("CvSize", "width,height")
m = m.replace("cvmatnd", "matND")
m = m.replace("CvSeqOfCvConvexityDefect", "convexityDefects")
mm = m.split(',')
if len(mm) > 1:
return "(" + ", ".join(mm) + ")"
else:
return m
docstring = re.sub(r"(?<=-> )(.*)$", retvalRplace, docstring)
docstring = docstring.replace("( [, ", "([")
if sign != docstring:
logerror(ERROR_006_INVALIDPYOLDDOC, "old-style documentation differs from pydoc\npydoc: " + docstring + "\nfixup: " + sign + "\ncvdoc: " + signature[1], doc)
elif signature[0] == "Python2":
pname = signature[1][4:signature[1].find('(')]
cvname = "cv." + pname
parent = None
for cl in pyclsnamespaces:
if cvname.startswith(cl + "."):
if cl.startswith(parent or ""):
parent = cl
try:
if parent:
instance, clsname = get_cv2_object(parent)
fn = getattr(instance, cvname[len(parent)+1:])
docstr = fn.__doc__
docprefix = "cv2." + clsname + "."
else:
fn = getattr(cv2, pname)
docstr = fn.__doc__
docprefix = "cv2."
except AttributeError:
if parent:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented member of " + parent + " class: cv2." + pname, doc)
else:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function cv2." + pname, doc)
signature.append(DOCUMENTED_MARKER) # stop subsequent errors
continue
docstrings = [docprefix + s.replace("([, ", "([") for s in docstr.split(" or ")]
if not signature[1] in docstrings:
pydocs = "\npydoc: ".join(docstrings)
logerror(ERROR_007_INVALIDPYDOC, "documentation differs from pydoc\npydoc: " + pydocs + "\ncvdoc: " + signature[1], doc)
signature.append(DOCUMENTED_MARKER)
# verify C/C++ signatures
for name, doc in rst.iteritems():
decls = doc.get("decls")
if not decls:
continue
for signature in decls:
if signature[0] == "C" or signature[0] == "C++":
if "template" in (signature[2][1] or ""):
# TODO find a way to validate templates
signature.append(DOCUMENTED_MARKER)
continue
fd = flookup.get(signature[2][0])
if not fd:
if signature[2][0].startswith("cv."):
fd = flookup.get(signature[2][0][3:])
if not fd:
continue
else:
signature[2][0] = signature[2][0][3:]
if signature[0] == "C":
ffd = [f for f in fd if not f[0] and not f[1]] # filter out C++ stuff
if not ffd:
if fd[0][1]:
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually member of " + fd[0][1] + " class", doc)
elif fd[0][0]:
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually placed in " + fd[0][0] + " namespace", doc)
fd = ffd
error = None
for f in fd:
match, error = compareSignatures(signature[2], f[3])
if match:
signature.append(DOCUMENTED_MARKER)
break
if signature[-1] != DOCUMENTED_MARKER:
candidates = "\n\t".join([formatSignature(f[3]) for f in fd])
logerror(ERROR_009_OVERLOADNOTFOUND, signature[0] + " function " + signature[2][0].replace(".","::") + " is documented but misses in headers (" + error + ").\nDocumented as:\n\t" + signature[1] + "\nCandidates are:\n\t" + candidates, doc)
signature.append(DOCUMENTED_MARKER) # to stop subsequent error on this function
# verify that all signatures was found in the library headers
for name, doc in rst.iteritems():
# if doc.get(DOCUMENTED_MARKER, False):
# continue # this class/struct was found
if not doc.get(DOCUMENTED_MARKER, False) and (doc.get("isclass", False) or doc.get("isstruct", False)):
if name in doc_signatures_whitelist:
continue
logerror(ERROR_010_UNKNOWNCLASS, "class/struct " + name + " is mentioned in documentation but is not found in OpenCV headers", doc)
for d in doc.get("decls", []):
if d[-1] != DOCUMENTED_MARKER:
if d[0] == "C" or d[0] =="C++" or (do_python_crosscheck and d[0].startswith("Python")):
if d[0][0] == 'C':
sname = d[2][0][3:].replace(".", "::")
if sname in defines:
#TODO: need to find a way to verify #define's
continue
else:
sname = d[1][:d[1].find("(")]
prefixes = [x for x in doc_signatures_whitelist if sname.startswith(x)]
if prefixes:
# TODO: member of template class
continue
logerror(ERROR_011_UNKNOWNFUNC, d[0] + " function " + sname + " is documented but is not found in OpenCV headers. It is documented as:\n\t" + d[1], doc)
def gen(self, srcfiles, output_path):
self.clear()
self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True,
generate_gpumat_decls=True)
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = self.parser.parse(hdr)
if len(decls) == 0:
continue
if hdr.find('opencv2/') >= 0: #Avoid including the shadow files
self.code_include.write('#include "{0}"\n'.format(
hdr[hdr.rindex('opencv2/'):]))
for decl in decls:
name = decl[0]
if name.startswith("struct") or name.startswith("class"):
# class/struct
p = name.find(" ")
stype = name[:p]
name = name[p + 1:].strip()
self.add_class(stype, name, decl)
elif name.startswith("const"):
# constant
self.add_const(name.replace("const ", "").strip(), decl)
elif name.startswith("enum"):
# enum
self.add_enum(name.rsplit(" ", 1)[1], decl)
else:
# function
self.add_func(decl)
# step 1.5 check if all base classes exist
for name, classinfo in self.classes.items():
if classinfo.base:
chunks = classinfo.base.split('_')
base = '_'.join(chunks)
while base not in self.classes and len(chunks) > 1:
del chunks[-2]
base = '_'.join(chunks)
if base not in self.classes:
print("Generator error: unable to resolve base %s for %s" %
(classinfo.base, classinfo.name))
sys.exit(-1)
base_instance = self.classes[base]
classinfo.base = base
classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag:
# doesn't work for trees(graphs) with depth > 2
self.classes[name] = classinfo
# tree-based propagation of 'isalgorithm'
processed = dict()
def process_isalgorithm(classinfo):
if classinfo.isalgorithm or classinfo in processed:
return classinfo.isalgorithm
res = False
if classinfo.base:
res = process_isalgorithm(self.classes[classinfo.base])
#assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base
classinfo.isalgorithm |= res
res = classinfo.isalgorithm
processed[classinfo] = True
return res
for name, classinfo in self.classes.items():
process_isalgorithm(classinfo)
# step 2: generate code for the classes and their methods
classlist = list(self.classes.items())
classlist.sort()
for name, classinfo in classlist:
self.code_types.write("//{}\n".format(80 * "="))
self.code_types.write("// {} ({})\n".format(
name, 'Map' if classinfo.ismap else 'Generic'))
self.code_types.write("//{}\n".format(80 * "="))
self.code_types.write(classinfo.gen_code(self))
if classinfo.ismap:
self.code_types.write(
gen_template_map_type_cvt.substitute(
name=classinfo.name, cname=classinfo.cname))
else:
mappable_code = "\n".join([
gen_template_mappable.substitute(cname=classinfo.cname,
mappable=mappable)
for mappable in classinfo.mappables
])
code = gen_template_type_decl.substitute(
name=classinfo.name,
cname=classinfo.cname if classinfo.issimple else
"Ptr<{}>".format(classinfo.cname),
mappable_code=mappable_code)
self.code_types.write(code)
# register classes in the same order as they have been declared.
# this way, base classes will be registered in Python before their derivatives.
classlist1 = [(classinfo.decl_idx, name, classinfo)
for name, classinfo in classlist]
classlist1.sort()
for decl_idx, name, classinfo in classlist1:
if classinfo.ismap:
continue
self.code_type_publish.write(classinfo.gen_def(self))
# step 3: generate the code for all the global functions
for ns_name, ns in sorted(self.namespaces.items()):
if ns_name.split('.')[0] != 'cv':
continue
for name, func in sorted(ns.funcs.items()):
if func.isconstructor:
continue
code = func.gen_code(self)
self.code_funcs.write(code)
self.gen_namespace(ns_name)
self.code_ns_init.write('CVPY_MODULE("{}", {});\n'.format(
ns_name[2:], normalize_class_name(ns_name)))
# step 4: generate the code for enum types
enumlist = list(self.enums.values())
enumlist.sort()
for name in enumlist:
self.gen_enum_reg(name)
# step 5: generate the code for constants
constlist = list(self.consts.items())
constlist.sort()
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# That's it. Now save all the files
self.save(output_path, "pyopencv_generated_include.h",
self.code_include)
self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs)
self.save(output_path, "pyopencv_generated_enums.h", self.code_enums)
self.save(output_path, "pyopencv_generated_types.h",
self.code_type_publish)
self.save(output_path, "pyopencv_generated_types_content.h",
self.code_types)
self.save(output_path, "pyopencv_generated_modules.h",
self.code_ns_init)
self.save(output_path, "pyopencv_generated_modules_content.h",
self.code_ns_reg)
self.save_json(output_path, "pyopencv_signatures.json",
self.py_signatures)
import rst_parser
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="Print verbose log to stdout", action="store_true", default=False)
parser.add_option("", "--no-warnings", dest="warnings", help="Hide warning messages", action="store_false", default=True)
parser.add_option("", "--no-errors", dest="errors", help="Hide error messages", action="store_false", default=True)
parser.add_option("", "--modules", dest="modules", help="comma-separated list of modules to generate comments", metavar="MODS", default=",".join(rst_parser.allmodules))
(options, args) = parser.parse_args(sys.argv)
options.modules = options.modules.split(",")
if len(args) < 2 or len(options.modules) < 1:
parser.print_help()
exit(0)
parser = rst_parser.RstParser(hdr_parser.CppHeaderParser())
for m in options.modules:
parser.parse(m, os.path.join(selfpath, "../../" + m))
parser.printSummary()
generator = JavadocGenerator(parser.definitions, options.modules)
generator.verbose = options.verbose
generator.show_warnings = options.warnings
generator.show_errors = options.errors
for path in args:
folder = os.path.abspath(path)
for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]:
outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java"))
generator.document(jfile, outfile)
def gen(self, srcfiles, output_path):
self.clear()
parser = hdr_parser.CppHeaderParser()
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = parser.parse(hdr)
for decl in decls:
name = decl[0]
if name.startswith("struct") or name.startswith("class"):
# class/struct
p = name.find(" ")
stype = name[:p]
name = name[p + 1:].strip()
self.add_class(stype, name, decl)
elif name.startswith("const"):
# constant
self.add_const(name.replace("const ", "").strip(), decl)
else:
# function
self.add_func(decl)
# step 2: generate code for the classes and their methods
classlist = self.classes.items()
classlist.sort()
for name, classinfo in classlist:
if classinfo.ismap:
self.code_types.write(
gen_template_map_type_cvt.substitute(
name=name, cname=classinfo.cname))
else:
if classinfo.issimple:
templ = gen_template_simple_type_decl
else:
templ = gen_template_type_decl
self.code_types.write(
templ.substitute(
name=name,
wname=classinfo.wname,
cname=classinfo.cname,
cname1=("cv::Algorithm" if classinfo.isalgorithm else
classinfo.cname)))
# register classes in the same order as they have been declared.
# this way, base classes will be registered in Python before their derivatives.
classlist1 = [(classinfo.decl_idx, name, classinfo)
for name, classinfo in classlist]
classlist1.sort()
for decl_idx, name, classinfo in classlist1:
code = classinfo.gen_code(self.classes)
self.code_types.write(code)
if not classinfo.ismap:
self.code_type_reg.write("MKTYPE2(%s);\n" % (classinfo.name, ))
# step 3: generate the code for all the global functions
funclist = self.funcs.items()
funclist.sort()
for name, func in funclist:
code = func.gen_code(self.classes)
self.code_funcs.write(code)
self.code_func_tab.write(func.get_tab_entry())
# step 4: generate the code for constants
constlist = self.consts.items()
constlist.sort()
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# That's it. Now save all the files
self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs)
self.save(output_path, "pyopencv_generated_func_tab.h",
self.code_func_tab)
self.save(output_path, "pyopencv_generated_const_reg.h",
self.code_const_reg)
self.save(output_path, "pyopencv_generated_types.h", self.code_types)
self.save(output_path, "pyopencv_generated_type_reg.h",
self.code_type_reg)
def gen_tree(srcfiles):
parser = hdr_parser.CppHeaderParser(generate_umat_decls=False,
generate_gpumat_decls=False)
allowed_func_list = []
with open("funclist.csv", "r") as f:
allowed_func_list = f.readlines()
allowed_func_list = [x[:-1] for x in allowed_func_list]
count = 0
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = parser.parse(hdr)
for ns in parser.namespaces:
ns = ns.replace('.', '::')
if ns not in namespaces:
namespaces[ns] = NameSpaceInfo(ns)
count += len(decls)
if len(decls) == 0:
continue
if hdr.find('opencv2/') >= 0: #Avoid including the shadow files
# code_include.write( '#include "{0}"\n'.format(hdr[hdr.rindex('opencv2/'):]) )
pass
for decl in decls:
name = decl[0]
if name.startswith("struct") or name.startswith("class"):
# class/struct
p = name.find(" ")
stype = name[:p]
name = name[p + 1:].strip()
add_class(stype, name, decl)
elif name.startswith("const"):
# constant
assert (0)
add_const(name.replace("const ", "").strip(), decl)
elif name.startswith("enum"):
# enum
add_enum(name.rsplit(" ", 1)[1], decl)
else:
# function
if decl[0] in allowed_func_list:
add_func(decl)
# step 1.5 check if all base classes exist
# print(classes)
for name, classinfo in classes.items():
if classinfo.base:
base = classinfo.base
# print(base)
if base not in classes:
print("Generator error: unable to resolve base %s for %s" %
(classinfo.base, classinfo.name))
sys.exit(-1)
base_instance = classes[base]
classinfo.base = base
classinfo.isalgorithm |= base_instance.isalgorithm # wrong processing of 'isalgorithm' flag:
# doesn't work for trees(graphs) with depth > 2
classes[name] = classinfo
# tree-based propagation of 'isalgorithm'
processed = dict()
def process_isalgorithm(classinfo):
if classinfo.isalgorithm or classinfo in processed:
return classinfo.isalgorithm
res = False
if classinfo.base:
res = process_isalgorithm(classes[classinfo.base])
#assert not (res == True or classinfo.isalgorithm is False), "Internal error: " + classinfo.name + " => " + classinfo.base
classinfo.isalgorithm |= res
res = classinfo.isalgorithm
processed[classinfo] = True
return res
for name, classinfo in classes.items():
process_isalgorithm(classinfo)
for name, ns in namespaces.items():
if name.split('.')[-1] == '':
continue
ns.registered = []
for name, cl in ns.classes.items():
registered_types.append(get_template_arg(name))
ns.registered.append(cl.mapped_name)
nss, clss, bs = split_decl_name(name)
type_paths[bs] = [name.replace("::", ".")]
type_paths["::".join(clss + [bs])] = [name.replace("::", ".")]
for e1, e2 in ns.enums.items():
registered_types.append(get_template_arg(e2[0]))
registered_types.append(
get_template_arg(e2[0]).replace('::', '_')) #whyyy typedef
ns.registered.append(e2[1])
ns.register_types = list(set(ns.register_types))
ns.register_types = [
tp for tp in ns.register_types
if not registered_tp_search(tp) and not tp in ns.registered
]
for tp in ns.register_types:
registered_types.append(get_template_arg(tp))
ns.registered.append(get_template_arg(tp))
default_valuesr = list(set(default_values))
# registered_types = registered_types + ns.register_types
return namespaces, default_valuesr
def gen(self, module_roots, modules, extras, output_dir):
"""
Generate a set of Matlab mex source files by parsing exported symbols
in a set of C++ headers. The headers can be input in one (or both) of
two methods:
1. specify module_root and modules
Given a path to the OpenCV module root and a list of module names,
the headers to parse are implicitly constructed.
2. specifiy header locations explicitly in extras
Each element in the list of extras must be of the form:
'namespace=/full/path/to/extra/header.hpp' where 'namespace' is
the namespace in which the definitions should be added.
The output_dir specifies the directory to write the generated sources
to.
"""
# dynamically import the parsers
from jinja2 import Environment, FileSystemLoader
import hdr_parser
import rst_parser
# parse each of the files and store in a dictionary
# as a separate "namespace"
parser = hdr_parser.CppHeaderParser()
rst = rst_parser.RstParser(parser)
rst_parser.verbose = False
rst_parser.show_warnings = False
rst_parser.show_errors = False
rst_parser.show_critical_errors = False
ns = dict((key, []) for key in modules)
doc = dict((key, []) for key in modules)
path_template = Template('${module}/include/opencv2/${module}.hpp')
for module in modules:
for module_root in module_roots:
# construct a header path from the module root and a path template
header = os.path.join(module_root, path_template.substitute(module=module))
if os.path.isfile(header):
break
else:
raise Exception('no header found for module %s!' % module)
# parse the definitions
ns[module] = parser.parse(header)
# parse the documentation
rst.parse(module, os.path.join(module_root, module))
doc[module] = rst.definitions
rst.definitions = {}
for extra in extras:
module = extra.split("=")[0]
header = extra.split("=")[1]
ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header)
# cleanify the parser output
parse_tree = ParseTree()
parse_tree.build(ns)
# setup the template engine
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True)
# add the custom filters
jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant
jtemplate.filters['convertibleToInt'] = convertibleToInt
jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase
jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase
jtemplate.filters['toUnderCase'] = toUnderCase
jtemplate.filters['matlabURL'] = matlabURL
jtemplate.filters['stripTags'] = stripTags
jtemplate.filters['filename'] = filename
jtemplate.filters['comment'] = comment
jtemplate.filters['inputs'] = inputs
jtemplate.filters['ninputs'] = ninputs
jtemplate.filters['outputs'] = outputs
jtemplate.filters['noutputs'] = noutputs
jtemplate.filters['qualify'] = qualify
jtemplate.filters['slugify'] = slugify
jtemplate.filters['only'] = only
jtemplate.filters['void'] = void
jtemplate.filters['not'] = flip
# load the templates
tfunction = jtemplate.get_template('template_function_base.cpp')
tclassm = jtemplate.get_template('template_class_base.m')
tclassc = jtemplate.get_template('template_class_base.cpp')
tdoc = jtemplate.get_template('template_doc_base.m')
tconst = jtemplate.get_template('template_map_base.m')
# create the build directory
output_source_dir = output_dir+'/src'
output_private_dir = output_source_dir+'/private'
output_class_dir = output_dir+'/+cv'
output_map_dir = output_dir+'/map'
if not os.path.isdir(output_source_dir):
os.makedirs(output_source_dir)
if not os.path.isdir(output_private_dir):
os.makedirs(output_private_dir)
if not os.path.isdir(output_class_dir):
os.makedirs(output_class_dir)
if not os.path.isdir(output_map_dir):
os.makedirs(output_map_dir)
# populate templates
for namespace in parse_tree.namespaces:
# functions
for method in namespace.methods:
populated = tfunction.render(fun=method, time=time, includes=namespace.name)
with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
if namespace.name in doc and method.name in doc[namespace.name]:
populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time)
with open(output_class_dir+'/'+method.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# classes
for clss in namespace.classes:
# cpp converter
populated = tclassc.render(clss=clss, time=time)
with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
# matlab classdef
populated = tclassm.render(clss=clss, time=time)
with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# create a global constants lookup table
const = dict(constants(todict(parse_tree.namespaces)))
populated = tconst.render(constants=const, time=time)
with open(output_dir+'/cv.m', 'wb') as f:
f.write(populated.encode('utf-8'))
if len(sys.argv) > 2:
system_include_dir_search.insert(0, sys.argv[2])
system_include_dir = None
for include_dir in system_include_dir_search:
if os.path.exists(os.path.join(include_dir, src_files[0])):
system_include_dir = include_dir
break
print('Using include dir: {}'.format(system_include_dir))
# TODO enable UMat support, and make a wrapper for handling both Mat and UMat identically
generate_umat = False
parser = hdr_parser.CppHeaderParser(generate_umat_decls=generate_umat,
generate_gpumat_decls=generate_umat)
functions = []
enums = []
enum_map = {}
defined_enum_consts = set()
classes = {}
overload_counts = {}
ocaml_overload_counts = {}
draw_functions = []
def add_struct(struct):
type_manager.add_type(
type_manager.CustomType(
struct.cpp_name,
struct.cpp_name,
'unit ptr',
