def extract_data_doc(state: State, parent, path: List[str], data):
assert not inspect.ismodule(data) and not inspect.isclass(data) and not inspect.isroutine(data) and not inspect.isframe(data) and not inspect.istraceback(data) and not inspect.iscode(data)
out = Empty()
out.name = path[-1]
# Welp. https://stackoverflow.com/questions/8820276/docstring-for-variable
out.summary = ''
out.has_details = False
if hasattr(parent, '__annotations__') and out.name in parent.__annotations__:
out.type = extract_annotation(state, parent.__annotations__[out.name])
else:
out.type = None
# The autogenerated <foo.bar at 0xbadbeef> is useless, so provide the value
# only if __repr__ is implemented for given type
if '__repr__' in type(data).__dict__:
out.value = html.escape(repr(data))
else:
out.value = None
# External data summary, if provided
path_str = '.'.join(path)
if path_str in state.data_docs:
# TODO: use also the contents
out.summary = render_inline_rst(state, state.data_docs[path_str]['summary'])
del state.data_docs[path_str]
return out
def extract_class_doc(state: State, path: List[str], class_):
assert inspect.isclass(class_)
out = Empty()
out.url = make_url(path)
out.name = path[-1]
out.summary = extract_summary(state, state.class_docs, path, class_.__doc__)
return out
def extract_module_doc(state: State, path: List[str], module):
assert inspect.ismodule(module)
out = Empty()
out.url = make_url(path)
out.name = path[-1]
out.summary = extract_summary(state, state.class_docs, path, module.__doc__)
return out
def render_page(state: State, path, filename, env):
logging.debug("generating %s.html", '.'.join(path))
# Call all registered page begin hooks
for hook in state.hooks_pre_page: hook()
# Render the file
with open(filename, 'r') as f: pub = publish_rst(state, f.read())
# Extract metadata from the page
metadata = {}
for docinfo in pub.document.traverse(docutils.nodes.docinfo):
for element in docinfo.children:
if element.tagname == 'field':
name_elem, body_elem = element.children
name = name_elem.astext()
if name in state.config['FORMATTED_METADATA']:
# If the metadata are formatted, format them. Use a special
# translator that doesn't add <dd> tags around the content,
# also explicitly disable the <p> around as we not need it
# always.
# TODO: uncrapify this a bit
visitor = m.htmlsanity._SaneFieldBodyTranslator(pub.document)
visitor.compact_field_list = True
body_elem.walkabout(visitor)
value = visitor.astext()
else:
value = body_elem.astext()
metadata[name.lower()] = value
# Breadcrumb, we don't do page hierarchy yet
assert len(path) == 1
breadcrumb = [(pub.writer.parts.get('title'), path[0] + '.html')]
page = Empty()
page.url = breadcrumb[-1][1]
page.breadcrumb = breadcrumb
page.prefix_wbr = path[0]
# Set page content and add extra metadata from there
page.content = pub.writer.parts.get('body').rstrip()
for key, value in metadata.items(): setattr(page, key, value)
if not hasattr(page, 'summary'): page.summary = ''
render(state.config, 'page.html', page, env)
# Index entry for this page, return only if it's not an index
if path == ['index']: return []
index_entry = IndexEntry()
index_entry.kind = 'page'
index_entry.name = breadcrumb[-1][0]
index_entry.url = page.url
index_entry.summary = page.summary
return [index_entry]
def extract_enum_doc(state: State, path: List[str], enum_):
out = Empty()
out.name = path[-1]
out.values = []
out.has_details = False
out.has_value_details = False
# The happy case
if issubclass(enum_, enum.Enum):
# Enum doc is by default set to a generic value. That's useless as well.
if enum_.__doc__ == 'An enumeration.':
out.summary = ''
else:
# TODO: external summary for enums
out.summary = extract_summary(state, {}, [], enum_.__doc__)
out.base = extract_type(enum_.__base__)
for i in enum_:
value = Empty()
value.name = i.name
value.value = html.escape(repr(i.value))
# Value doc gets by default inherited from the enum, that's useless
if i.__doc__ == enum_.__doc__:
value.summary = ''
else:
# TODO: external summary for enum values
value.summary = extract_summary(state, {}, [], i.__doc__)
if value.summary:
out.has_details = True
out.has_value_details = True
out.values += [value]
# Pybind11 enums are ... different
elif state.config['PYBIND11_COMPATIBILITY']:
assert hasattr(enum_, '__members__')
# TODO: external summary for enums
out.summary = extract_summary(state, {}, [], enum_.__doc__)
out.base = None
for name, v in enum_.__members__.items():
value = Empty()
value. name = name
value.value = int(v)
# TODO: once https://github.com/pybind/pybind11/pull/1160 is
# released, extract from class docs (until then the class
# docstring is duplicated here, which is useless)
value.summary = ''
out.values += [value]
return out
def extract_property_doc(state: State, path: List[str], property):
assert inspect.isdatadescriptor(property)
out = Empty()
out.name = path[-1]
# TODO: external summary for properties
out.summary = extract_summary(state, {}, [], property.__doc__)
out.is_settable = property.fset is not None
out.is_deletable = property.fdel is not None
out.has_details = False
try:
signature = inspect.signature(property.fget)
out.type = extract_annotation(state, signature.return_annotation)
except ValueError:
# pybind11 properties have the type in the docstring
if state.config['PYBIND11_COMPATIBILITY']:
out.type = parse_pybind_signature(state, property.fget.__doc__)[3]
else:
out.type = None
return out
def render_class(state: State, path, class_, env):
logging.debug("generating %s.html", '.'.join(path))
# Call all registered page begin hooks
for hook in state.hooks_pre_page: hook()
url_base = ''
breadcrumb = []
for i in path:
url_base += i + '.'
breadcrumb += [(i, url_base + 'html')]
page = Empty()
page.summary = extract_summary(state, state.class_docs, path, class_.__doc__)
page.url = breadcrumb[-1][1]
page.breadcrumb = breadcrumb
page.prefix_wbr = '.<wbr />'.join(path + [''])
page.classes = []
page.enums = []
page.classmethods = []
page.staticmethods = []
page.dunder_methods = []
page.methods = []
page.properties = []
page.data = []
page.has_enum_details = False
# External page content, if provided
path_str = '.'.join(path)
if path_str in state.class_docs:
page.content = render_rst(state, state.class_docs[path_str]['content'])
state.class_docs[path_str]['used'] = True
# Index entry for this module, returned together with children at the end
index_entry = IndexEntry()
index_entry.kind = 'class'
index_entry.name = breadcrumb[-1][0]
index_entry.url = page.url
index_entry.summary = page.summary
# List of inner classes to render, these will be done after the current
# class introspection is done to have some better memory allocation pattern
classes_to_render = []
# Get inner classes
for name, object in inspect.getmembers(class_, lambda o: inspect.isclass(o) and not is_enum(state, o)):
if name in ['__base__', '__class__']: continue # TODO
if name.startswith('_'): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.classes += [extract_class_doc(state, subpath, object)]
classes_to_render += [(subpath, object)]
# Get enums
for name, object in inspect.getmembers(class_, lambda o: is_enum(state, o)):
if name.startswith('_'): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
# Get methods
for name, object in inspect.getmembers(class_, inspect.isroutine):
# Filter out underscored methods (but not dunder methods)
if is_internal_function_name(name): continue
# Filter out dunder methods that don't have their own docs
if name.startswith('__') and (name, object.__doc__) in _filtered_builtin_functions: continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s() is undocumented", '.'.join(subpath))
for function in extract_function_doc(state, class_, subpath, object):
if name.startswith('__'):
page.dunder_methods += [function]
elif function.is_classmethod:
page.classmethods += [function]
elif function.is_staticmethod:
page.staticmethods += [function]
else:
page.methods += [function]
# Get properties
for name, object in inspect.getmembers(class_, inspect.isdatadescriptor):
if (name, object.__doc__) in _filtered_builtin_properties:
continue
if name.startswith('_'): continue # TODO: are there any dunder props?
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.properties += [extract_property_doc(state, subpath, object)]
# Get data
# TODO: unify this query
for name, object in inspect.getmembers(class_, lambda o: not inspect.ismodule(o) and not inspect.isclass(o) and not inspect.isroutine(o) and not inspect.isframe(o) and not inspect.istraceback(o) and not inspect.iscode(o) and not inspect.isdatadescriptor(o)):
if name.startswith('_'): continue
subpath = path + [name]
page.data += [extract_data_doc(state, class_, subpath, object)]
# Render the class, free the page data to avoid memory rising indefinitely
render(state.config, 'class.html', page, env)
del page
# Render subclasses
for subpath, object in classes_to_render:
index_entry.children += [render_class(state, subpath, object, env)]
return index_entry
def render_module(state: State, path, module, env):
logging.debug("generating %s.html", '.'.join(path))
# Call all registered page begin hooks
for hook in state.hooks_pre_page: hook()
url_base = ''
breadcrumb = []
for i in path:
url_base += i + '.'
breadcrumb += [(i, url_base + 'html')]
page = Empty()
page.summary = extract_summary(state, state.module_docs, path, module.__doc__)
page.url = breadcrumb[-1][1]
page.breadcrumb = breadcrumb
page.prefix_wbr = '.<wbr />'.join(path + [''])
page.modules = []
page.classes = []
page.enums = []
page.functions = []
page.data = []
page.has_enum_details = False
# External page content, if provided
path_str = '.'.join(path)
if path_str in state.module_docs:
page.content = render_rst(state, state.module_docs[path_str]['content'])
state.module_docs[path_str]['used'] = True
# Index entry for this module, returned together with children at the end
index_entry = IndexEntry()
index_entry.kind = 'module'
index_entry.name = breadcrumb[-1][0]
index_entry.url = page.url
index_entry.summary = page.summary
# List of inner modules and classes to render, these will be done after the
# current class introspection is done to have some better memory allocation
# pattern
modules_to_render = []
classes_to_render = []
# This is actually complicated -- if the module defines __all__, use that.
# The __all__ is meant to expose the public API, so we don't filter out
# underscored things.
if hasattr(module, '__all__'):
# Names exposed in __all__ could be also imported from elsewhere, for
# example this is a common pattern with native libraries and we want
# Foo, Bar, submodule and *everything* in submodule to be referred to
# as `library.RealName` (`library.submodule.func()`, etc.) instead of
# `library._native.Foo`, `library._native.sub.func()` etc.
#
# from ._native import Foo as PublicName
# from ._native import sub as submodule
# __all__ = ['PublicName', 'submodule']
#
# The name references can be cyclic so extract the mapping in a
# separate pass before everything else.
for name in module.__all__:
# Everything available in __all__ is already imported, so get those
# directly
object = getattr(module, name)
subpath = path + [name]
# Modules have __name__ while other objects have __module__, need
# to check both.
if inspect.ismodule(object) and object.__name__ != '.'.join(subpath):
assert object.__name__ not in state.module_mapping
state.module_mapping[object.__name__] = '.'.join(subpath)
elif hasattr(object, '__module__'):
subname = object.__module__ + '.' + object.__name__
if subname != '.'.join(subpath):
assert subname not in state.module_mapping
state.module_mapping[subname] = '.'.join(subpath)
# Now extract the actual docs
for name in module.__all__:
object = getattr(module, name)
subpath = path + [name]
# We allow undocumented submodules (since they're often in the
# standard lib), but not undocumented classes etc. Render the
# submodules and subclasses recursively.
if inspect.ismodule(object):
page.modules += [extract_module_doc(state, subpath, object)]
index_entry.children += [render_module(state, subpath, object, env)]
elif inspect.isclass(object) and not is_enum(state, object):
page.classes += [extract_class_doc(state, subpath, object)]
index_entry.children += [render_class(state, subpath, object, env)]
elif inspect.isclass(object) and is_enum(state, object):
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
elif inspect.isfunction(object) or inspect.isbuiltin(object):
page.functions += extract_function_doc(state, module, subpath, object)
# Assume everything else is data. The builtin help help() (from
# pydoc) does the same:
# https://github.com/python/cpython/blob/d29b3dd9227cfc4a23f77e99d62e20e063272de1/Lib/pydoc.py#L113
# TODO: unify this query
elif not inspect.isframe(object) and not inspect.istraceback(object) and not inspect.iscode(object):
page.data += [extract_data_doc(state, module, subpath, object)]
else: # pragma: no cover
logging.warning("unknown symbol %s in %s", name, '.'.join(path))
# Otherwise, enumerate the members using inspect. However, inspect lists
# also imported modules, functions and classes, so take only those which
# have __module__ equivalent to `path`.
else:
# Get (and render) inner modules
for name, object in inspect.getmembers(module, inspect.ismodule):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
page.modules += [extract_module_doc(state, subpath, object)]
modules_to_render += [(subpath, object)]
# Get (and render) inner classes
for name, object in inspect.getmembers(module, lambda o: inspect.isclass(o) and not is_enum(state, o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.classes += [extract_class_doc(state, subpath, object)]
classes_to_render += [(subpath, object)]
# Get enums
for name, object in inspect.getmembers(module, lambda o: is_enum(state, o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
# Get inner functions
for name, object in inspect.getmembers(module, lambda o: inspect.isfunction(o) or inspect.isbuiltin(o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s() is undocumented", '.'.join(subpath))
page.functions += extract_function_doc(state, module, subpath, object)
# Get data
# TODO: unify this query
for name, object in inspect.getmembers(module, lambda o: not inspect.ismodule(o) and not inspect.isclass(o) and not inspect.isroutine(o) and not inspect.isframe(o) and not inspect.istraceback(o) and not inspect.iscode(o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
page.data += [extract_data_doc(state, module, path + [name], object)]
# Render the module, free the page data to avoid memory rising indefinitely
render(state.config, 'module.html', page, env)
del page
# Render submodules and subclasses
for subpath, object in modules_to_render:
index_entry.children += [render_module(state, subpath, object, env)]
for subpath, object in classes_to_render:
index_entry.children += [render_class(state, subpath, object, env)]
return index_entry
def extract_function_doc(state: State, parent, path: List[str], function) -> List[Any]:
assert inspect.isfunction(function) or inspect.ismethod(function) or inspect.isroutine(function)
# Extract the signature from the docstring for pybind11, since it can't
# expose it to the metadata: https://github.com/pybind/pybind11/issues/990
# What's not solvable with metadata, however, are function overloads ---
# one function in Python may equal more than one function on the C++ side.
# To make the docs usable, list all overloads separately.
if state.config['PYBIND11_COMPATIBILITY'] and function.__doc__.startswith(path[-1]):
funcs = parse_pybind_docstring(state, path[-1], function.__doc__)
overloads = []
for name, summary, args, type in funcs:
out = Empty()
out.name = path[-1]
out.params = []
out.has_complex_params = False
out.has_details = False
# TODO: external summary for functions
out.summary = summary
# Don't show None return type for void functions
out.type = None if type == 'None' else type
# There's no other way to check staticmethods than to check for
# self being the name of first parameter :( No support for
# classmethods, as C++11 doesn't have that
out.is_classmethod = False
if inspect.isclass(parent) and args and args[0][0] == 'self':
out.is_staticmethod = False
else:
out.is_staticmethod = True
# Guesstimate whether the arguments are positional-only or
# position-or-keyword. It's either all or none. This is a brown
# magic, sorry.
# For instance methods positional-only argument names are either
# self (for the first argument) or arg(I-1) (for second
# argument and further). Also, the `self` argument is
# positional-or-keyword only if there are positional-or-keyword
# arguments afgter it, otherwise it's positional-only.
if inspect.isclass(parent) and not out.is_staticmethod:
assert args and args[0][0] == 'self'
positional_only = True
for i, arg in enumerate(args[1:]):
name, type, default = arg
if name != 'arg{}'.format(i):
positional_only = False
break
# For static methods or free functions positional-only arguments
# are argI.
else:
positional_only = True
for i, arg in enumerate(args):
name, type, default = arg
if name != 'arg{}'.format(i):
positional_only = False
break
for i, arg in enumerate(args):
name, type, default = arg
param = Empty()
param.name = name
# Don't include redundant type for the self argument
if name == 'self': param.type = None
else: param.type = type
param.default = html.escape(default or '')
if type or default: out.has_complex_params = True
# *args / **kwargs can still appear in the parsed signatures if
# the function accepts py::args / py::kwargs directly
if name == '*args':
param.name = 'args'
param.kind = 'VAR_POSITIONAL'
elif name == '**kwargs':
param.name = 'kwargs'
param.kind = 'VAR_KEYWORD'
else:
param.kind = 'POSITIONAL_ONLY' if positional_only else 'POSITIONAL_OR_KEYWORD'
out.params += [param]
overloads += [out]
return overloads
# Sane introspection path for non-pybind11 code
else:
out = Empty()
out.name = path[-1]
out.params = []
out.has_complex_params = False
out.has_details = False
# TODO: external summary for functions
out.summary = extract_summary(state, {}, [], function.__doc__)
# Decide if classmethod or staticmethod in case this is a method
if inspect.isclass(parent):
out.is_classmethod = inspect.ismethod(function)
out.is_staticmethod = out.name in parent.__dict__ and isinstance(parent.__dict__[out.name], staticmethod)
try:
signature = inspect.signature(function)
out.type = extract_annotation(state, signature.return_annotation)
for i in signature.parameters.values():
param = Empty()
param.name = i.name
param.type = extract_annotation(state, i.annotation)
if param.type:
out.has_complex_params = True
if i.default is inspect.Signature.empty:
param.default = None
else:
param.default = repr(i.default)
out.has_complex_params = True
param.kind = str(i.kind)
out.params += [param]
# In CPython, some builtin functions (such as math.log) do not provide
# metadata about their arguments. Source:
# https://docs.python.org/3/library/inspect.html#inspect.signature
except ValueError:
param = Empty()
param.name = '...'
param.name_type = param.name
out.params = [param]
out.type = None
return [out]
