import rst_parser
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="Print verbose log to stdout", action="store_true", default=False)
parser.add_option("", "--no-warnings", dest="warnings", help="Hide warning messages", action="store_false", default=True)
parser.add_option("", "--no-errors", dest="errors", help="Hide error messages", action="store_false", default=True)
parser.add_option("", "--modules", dest="modules", help="comma-separated list of modules to generate comments", metavar="MODS", default=",".join(rst_parser.allmodules))
(options, args) = parser.parse_args(sys.argv)
options.modules = options.modules.split(",")
if len(args) < 2 or len(options.modules) < 1:
parser.print_help()
exit(0)
parser = rst_parser.RstParser(hdr_parser.CppHeaderParser())
for m in options.modules:
parser.parse(m, os.path.join(selfpath, "../../" + m))
parser.printSummary()
generator = JavadocGenerator(parser.definitions, options.modules)
generator.verbose = options.verbose
generator.show_warnings = options.warnings
generator.show_errors = options.errors
for path in args:
folder = os.path.abspath(path)
for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]:
outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java"))
generator.document(jfile, outfile)
def process_module(module, path):
hppparser = hp.CppHeaderParser()
rstparser = rp.RstParser(hppparser)
rstparser.parse(module, path)
rst = rstparser.definitions
hdrlist = []
for root, dirs, files in os.walk(os.path.join(path, "include")):
for filename in fnmatch.filter(files, "*.h*"):
hdrlist.append(os.path.join(root, filename))
if module == "gpu":
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_types.hpp"))
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu.hpp"))
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpu_stream_accessor.hpp"))
decls = []
for hname in hdrlist:
if not "ts_gtest.h" in hname:
decls += hppparser.parse(hname, wmode=False)
funcs = []
# not really needed to hardcode all the namespaces. Normally all they are collected automatically
namespaces = ['cv', 'cv.gpu', 'cvflann', 'cvflann.anyimpl', 'cvflann.lsh', 'cv.flann', 'cv.linemod', 'cv.detail', 'cvtest', 'perf', 'cv.videostab']
classes = []
structs = []
# collect namespaces and classes/structs
for decl in decls:
if decl[0].startswith("const"):
pass
elif decl[0].startswith("class") or decl[0].startswith("struct"):
if decl[0][0] == 'c':
classes.append(decl)
else:
structs.append(decl)
dotIdx = decl[0].rfind('.')
if dotIdx > 0:
namespace = decl[0][decl[0].find(' ')+1:dotIdx]
if not [c for c in classes if c[0].endswith(namespace)] and not [s for s in structs if s[0].endswith(namespace)]:
if namespace not in namespaces:
namespaces.append(namespace)
else:
funcs.append(decl)
clsnamespaces = []
# process classes
for cl in classes:
name = cl[0][cl[0].find(' ')+1:]
if name.find('.') < 0 and not name.startswith("Cv"):
logerror(ERROR_004_MISSEDNAMESPACE, "class " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
clsnamespaces.append(name)
if do_python_crosscheck and not name.startswith("cv.") and name.startswith("Cv"):
clsnamespaces.append("cv." + name[2:])
if name.startswith("cv."):
name = name[3:]
name = name.replace(".", "::")
sns = synonims.get(name, [])
sns.append(name)
for name in sns:
doc = rst.get(name)
if not doc:
#TODO: class is not documented
continue
doc[DOCUMENTED_MARKER] = True
# verify class marker
if not doc.get("isclass"):
logerror(ERROR_001_NOTACLASS, "class " + name + " is not marked as \"class\" in documentation", doc)
else:
# verify base
signature = doc.get("class", "")
signature = signature.replace(" public ", " ")
namespaceIdx = signature.rfind("::")
signature = ("class " + signature).strip()
hdrsignature = ("class " + name + " " + cl[1]).replace(".", "::").replace("cv::","").strip()
if signature != hdrsignature:
logerror(ERROR_003_INCORRECTBASE, "invalid base class documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
# process structs
for st in structs:
name = st[0][st[0].find(' ')+1:]
if name.find('.') < 0 and not name.startswith("Cv"):
logerror(ERROR_004_MISSEDNAMESPACE, "struct " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
clsnamespaces.append(name)
if name.startswith("cv."):
name = name[3:]
name = name.replace(".", "::")
doc = rst.get(name)
if not doc:
#TODO: struct is not documented
continue
doc[DOCUMENTED_MARKER] = True
# verify struct marker
if not doc.get("isstruct"):
logerror(ERROR_002_NOTASTRUCT, "struct " + name + " is not marked as \"struct\" in documentation", doc)
else:
# verify base
signature = doc.get("class", "")
signature = signature.replace(", public ", " ").replace(" public ", " ")
signature = signature.replace(", protected ", " ").replace(" protected ", " ")
signature = signature.replace(", private ", " ").replace(" private ", " ")
signature = ("struct " + signature).strip()
hdrsignature = (st[0] + " " + st[1]).replace("struct cv.", "struct ").replace(".", "::").strip()
if signature != hdrsignature:
logerror(ERROR_003_INCORRECTBASE, "invalid base struct documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
print st, doc
# process functions and methods
flookup = {}
for fn in funcs:
name = fn[0]
parent = None
namespace = None
for cl in clsnamespaces:
if name.startswith(cl + "."):
if cl.startswith(parent or ""):
parent = cl
if parent:
name = name[len(parent) + 1:]
for nm in namespaces:
if parent.startswith(nm + "."):
if nm.startswith(namespace or ""):
namespace = nm
if namespace:
parent = parent[len(namespace) + 1:]
else:
for nm in namespaces:
if name.startswith(nm + "."):
if nm.startswith(namespace or ""):
namespace = nm
if namespace:
name = name[len(namespace) + 1:]
#print namespace, parent, name, fn[0]
if not namespace and not parent and not name.startswith("cv") and not name.startswith("icv") and not name.startswith("CV_"):
logerror(ERROR_004_MISSEDNAMESPACE, "function " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
else:
fdescr = (namespace, parent, name, fn)
flookup_entry = flookup.get(fn[0], [])
flookup_entry.append(fdescr)
flookup[fn[0]] = flookup_entry
if do_python_crosscheck:
pyclsnamespaces = ["cv." + x[3:].replace(".", "_") for x in clsnamespaces]
for name, doc in rst.iteritems():
decls = doc.get("decls")
if not decls:
continue
for signature in decls:
if signature[0] == "Python1":
pname = signature[1][:signature[1].find('(')]
try:
fn = getattr(cv2.cv, pname[3:])
docstr = "cv." + fn.__doc__
except AttributeError:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function: cv2." + pname, doc)
continue
docstring = docstr
sign = signature[1]
signature.append(DOCUMENTED_MARKER)
# convert old signature to pydoc style
if docstring.endswith("*"):
docstring = docstring[:-1]
s = None
while s != sign:
s = sign
sign = re.sub(r"^(.*\(.*)\(.*?\)(.*\) *->)", "\\1_\\2", sign)
s = None
while s != sign:
s = sign
sign = re.sub(r"\s*,\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", " [, \\1\\2])", sign)
sign = re.sub(r"\(\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", "([\\1\\2])", sign)
sign = re.sub(r"\)\s*->\s*", ") -> ", sign)
sign = sign.replace("-> convexHull", "-> CvSeq")
sign = sign.replace("-> lines", "-> CvSeq")
sign = sign.replace("-> boundingRects", "-> CvSeq")
sign = sign.replace("-> contours", "-> CvSeq")
sign = sign.replace("-> retval", "-> int")
sign = sign.replace("-> detectedObjects", "-> CvSeqOfCvAvgComp")
def retvalRplace(match):
m = match.group(1)
m = m.replace("CvScalar", "scalar")
m = m.replace("CvMemStorage", "memstorage")
m = m.replace("ROIplImage", "image")
m = m.replace("IplImage", "image")
m = m.replace("ROCvMat", "mat")
m = m.replace("CvMat", "mat")
m = m.replace("double", "float")
m = m.replace("CvSubdiv2DPoint", "point")
m = m.replace("CvBox2D", "Box2D")
m = m.replace("IplConvKernel", "kernel")
m = m.replace("CvHistogram", "hist")
m = m.replace("CvSize", "width,height")
m = m.replace("cvmatnd", "matND")
m = m.replace("CvSeqOfCvConvexityDefect", "convexityDefects")
mm = m.split(',')
if len(mm) > 1:
return "(" + ", ".join(mm) + ")"
else:
return m
docstring = re.sub(r"(?<=-> )(.*)$", retvalRplace, docstring)
docstring = docstring.replace("( [, ", "([")
if sign != docstring:
logerror(ERROR_006_INVALIDPYOLDDOC, "old-style documentation differs from pydoc\npydoc: " + docstring + "\nfixup: " + sign + "\ncvdoc: " + signature[1], doc)
elif signature[0] == "Python2":
pname = signature[1][4:signature[1].find('(')]
cvname = "cv." + pname
parent = None
for cl in pyclsnamespaces:
if cvname.startswith(cl + "."):
if cl.startswith(parent or ""):
parent = cl
try:
if parent:
instance, clsname = get_cv2_object(parent)
fn = getattr(instance, cvname[len(parent)+1:])
docstr = fn.__doc__
docprefix = "cv2." + clsname + "."
else:
fn = getattr(cv2, pname)
docstr = fn.__doc__
docprefix = "cv2."
except AttributeError:
if parent:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented member of " + parent + " class: cv2." + pname, doc)
else:
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function cv2." + pname, doc)
signature.append(DOCUMENTED_MARKER) # stop subsequent errors
continue
docstrings = [docprefix + s.replace("([, ", "([") for s in docstr.split(" or ")]
if not signature[1] in docstrings:
pydocs = "\npydoc: ".join(docstrings)
logerror(ERROR_007_INVALIDPYDOC, "documentation differs from pydoc\npydoc: " + pydocs + "\ncvdoc: " + signature[1], doc)
signature.append(DOCUMENTED_MARKER)
# verify C/C++ signatures
for name, doc in rst.iteritems():
decls = doc.get("decls")
if not decls:
continue
for signature in decls:
if signature[0] == "C" or signature[0] == "C++":
if "template" in (signature[2][1] or ""):
# TODO find a way to validate templates
signature.append(DOCUMENTED_MARKER)
continue
fd = flookup.get(signature[2][0])
if not fd:
if signature[2][0].startswith("cv."):
fd = flookup.get(signature[2][0][3:])
if not fd:
continue
else:
signature[2][0] = signature[2][0][3:]
if signature[0] == "C":
ffd = [f for f in fd if not f[0] and not f[1]] # filter out C++ stuff
if not ffd:
if fd[0][1]:
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually member of " + fd[0][1] + " class", doc)
elif fd[0][0]:
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually placed in " + fd[0][0] + " namespace", doc)
fd = ffd
error = None
for f in fd:
match, error = compareSignatures(signature[2], f[3])
if match:
signature.append(DOCUMENTED_MARKER)
break
if signature[-1] != DOCUMENTED_MARKER:
candidates = "\n\t".join([formatSignature(f[3]) for f in fd])
logerror(ERROR_009_OVERLOADNOTFOUND, signature[0] + " function " + signature[2][0].replace(".","::") + " is documented but misses in headers (" + error + ").\nDocumented as:\n\t" + signature[1] + "\nCandidates are:\n\t" + candidates, doc)
signature.append(DOCUMENTED_MARKER) # to stop subsequent error on this function
# verify that all signatures was found in the library headers
for name, doc in rst.iteritems():
# if doc.get(DOCUMENTED_MARKER, False):
# continue # this class/struct was found
if not doc.get(DOCUMENTED_MARKER, False) and (doc.get("isclass", False) or doc.get("isstruct", False)):
if name in doc_signatures_whitelist:
continue
logerror(ERROR_010_UNKNOWNCLASS, "class/struct " + name + " is mentioned in documentation but is not found in OpenCV headers", doc)
for d in doc.get("decls", []):
if d[-1] != DOCUMENTED_MARKER:
if d[0] == "C" or d[0] =="C++" or (do_python_crosscheck and d[0].startswith("Python")):
if d[0][0] == 'C':
sname = d[2][0][3:].replace(".", "::")
if sname in defines:
#TODO: need to find a way to verify #define's
continue
else:
sname = d[1][:d[1].find("(")]
prefixes = [x for x in doc_signatures_whitelist if sname.startswith(x)]
if prefixes:
# TODO: member of template class
continue
logerror(ERROR_011_UNKNOWNFUNC, d[0] + " function " + sname + " is documented but is not found in OpenCV headers. It is documented as:\n\t" + d[1], doc)
def gen(self, module_roots, modules, extras, output_dir):
"""
Generate a set of Matlab mex source files by parsing exported symbols
in a set of C++ headers. The headers can be input in one (or both) of
two methods:
1. specify module_root and modules
Given a path to the OpenCV module root and a list of module names,
the headers to parse are implicitly constructed.
2. specifiy header locations explicitly in extras
Each element in the list of extras must be of the form:
'namespace=/full/path/to/extra/header.hpp' where 'namespace' is
the namespace in which the definitions should be added.
The output_dir specifies the directory to write the generated sources
to.
"""
# dynamically import the parsers
from jinja2 import Environment, FileSystemLoader
import hdr_parser
import rst_parser
# parse each of the files and store in a dictionary
# as a separate "namespace"
parser = hdr_parser.CppHeaderParser()
rst = rst_parser.RstParser(parser)
rst_parser.verbose = False
rst_parser.show_warnings = False
rst_parser.show_errors = False
rst_parser.show_critical_errors = False
ns = dict((key, []) for key in modules)
doc = dict((key, []) for key in modules)
path_template = Template('${module}/include/opencv2/${module}.hpp')
for module in modules:
for module_root in module_roots:
# construct a header path from the module root and a path template
header = os.path.join(module_root, path_template.substitute(module=module))
if os.path.isfile(header):
break
else:
raise Exception('no header found for module %s!' % module)
# parse the definitions
ns[module] = parser.parse(header)
# parse the documentation
rst.parse(module, os.path.join(module_root, module))
doc[module] = rst.definitions
rst.definitions = {}
for extra in extras:
module = extra.split("=")[0]
header = extra.split("=")[1]
ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header)
# cleanify the parser output
parse_tree = ParseTree()
parse_tree.build(ns)
# setup the template engine
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True)
# add the custom filters
jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant
jtemplate.filters['convertibleToInt'] = convertibleToInt
jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase
jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase
jtemplate.filters['toUnderCase'] = toUnderCase
jtemplate.filters['matlabURL'] = matlabURL
jtemplate.filters['stripTags'] = stripTags
jtemplate.filters['filename'] = filename
jtemplate.filters['comment'] = comment
jtemplate.filters['inputs'] = inputs
jtemplate.filters['ninputs'] = ninputs
jtemplate.filters['outputs'] = outputs
jtemplate.filters['noutputs'] = noutputs
jtemplate.filters['qualify'] = qualify
jtemplate.filters['slugify'] = slugify
jtemplate.filters['only'] = only
jtemplate.filters['void'] = void
jtemplate.filters['not'] = flip
# load the templates
tfunction = jtemplate.get_template('template_function_base.cpp')
tclassm = jtemplate.get_template('template_class_base.m')
tclassc = jtemplate.get_template('template_class_base.cpp')
tdoc = jtemplate.get_template('template_doc_base.m')
tconst = jtemplate.get_template('template_map_base.m')
# create the build directory
output_source_dir = output_dir+'/src'
output_private_dir = output_source_dir+'/private'
output_class_dir = output_dir+'/+cv'
output_map_dir = output_dir+'/map'
if not os.path.isdir(output_source_dir):
os.makedirs(output_source_dir)
if not os.path.isdir(output_private_dir):
os.makedirs(output_private_dir)
if not os.path.isdir(output_class_dir):
os.makedirs(output_class_dir)
if not os.path.isdir(output_map_dir):
os.makedirs(output_map_dir)
# populate templates
for namespace in parse_tree.namespaces:
# functions
for method in namespace.methods:
populated = tfunction.render(fun=method, time=time, includes=namespace.name)
with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
if namespace.name in doc and method.name in doc[namespace.name]:
populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time)
with open(output_class_dir+'/'+method.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# classes
for clss in namespace.classes:
# cpp converter
populated = tclassc.render(clss=clss, time=time)
with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
# matlab classdef
populated = tclassm.render(clss=clss, time=time)
with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# create a global constants lookup table
const = dict(constants(todict(parse_tree.namespaces)))
populated = tconst.render(constants=const, time=time)
with open(output_dir+'/cv.m', 'wb') as f:
f.write(populated.encode('utf-8'))
