def parse_docstring(api_doc, docindex):
"""
Process the given C{APIDoc}'s docstring. In particular, populate
the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
information provided by fields in the docstring.
@param docindex: A DocIndex, used to find the containing
module (to look up the docformat); and to find any
user docfields defined by containing objects.
"""
if api_doc.metadata is not UNKNOWN:
log.debug("%s's docstring processed twice" % api_doc.canonical_name)
return
initialize_api_doc(api_doc)
# If there's no docstring, then there's nothing more to do.
if (api_doc.docstring in (None, UNKNOWN)):
return
# Remove leading indentation from the docstring.
api_doc.docstring = unindent_docstring(api_doc.docstring)
# Extract a signature from the docstring, if it has one. This
# overrides any signature we got via introspection/parsing.
if isinstance(api_doc, RoutineDoc):
parse_function_signature(api_doc)
# Parse the docstring. Any errors encountered are stored as
# `ParseError` objects in the errors list.
docformat = get_docformat(api_doc, docindex)
parse_errors = []
parsed_docstring = markup.parse(api_doc.docstring, docformat,
parse_errors)
# Divide the docstring into a description and a list of
# fields.
descr, fields = parsed_docstring.split_fields(parse_errors)
api_doc.descr = descr
# Process fields
field_warnings = []
for field in fields:
try:
process_field(api_doc, docindex, field.tag(),
field.arg(), field.body())
except ValueError, e: field_warnings.append(str(e))
# Extract a summary
if api_doc.summary is None and api_doc.descr is not None:
api_doc.summary = api_doc.descr.summary()
# [XX] Make sure we don't have types/param descrs for unknown
# vars/params?
# Report any errors that occured
report_errors(api_doc, docindex, parse_errors, field_warnings)
def ctypesIntrospecter(value, value_doc):
"""Add ctypes documentation to the given value's documentation"""
value_doc.specialize_to(apidoc.RoutineDoc)
value_doc.is_imported = False
value_doc.is_public = True
if hasattr( value, 'argNames' ):
value_doc.posargs = value.argNames
value_doc.posarg_defaults = [None] * len(value.argNames)
if hasattr( value, '__doc__' ):
value_doc.docstring = docintrospecter.get_docstring(value)
value_doc.return_type = markup.parse( str(value.restype) )
return value_doc
def ctypesIntrospecter(value, value_doc):
"""Add ctypes documentation to the given value's documentation"""
value_doc.specialize_to(apidoc.RoutineDoc)
value_doc.is_imported = False
value_doc.is_public = True
if hasattr(value, 'argNames'):
value_doc.posargs = value.argNames
value_doc.posarg_defaults = [None] * len(value.argNames)
if hasattr(value, '__doc__'):
value_doc.docstring = docintrospecter.get_docstring(value)
value_doc.return_type = markup.parse(str(value.restype))
return value_doc
def parse_docstring(api_doc, docindex, suppress_warnings=[]):
"""
Process the given C{APIDoc}'s docstring. In particular, populate
the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
information provided by fields in the docstring.
@param docindex: A DocIndex, used to find the containing
module (to look up the docformat); and to find any
user docfields defined by containing objects.
@param suppress_warnings: A set of objects for which docstring
warnings should be suppressed.
"""
if api_doc.metadata is not UNKNOWN:
if not (isinstance(api_doc, RoutineDoc)
and api_doc.canonical_name[-1] == '__init__'):
log.debug("%s's docstring processed twice" %
api_doc.canonical_name)
return
initialize_api_doc(api_doc)
# If there's no docstring, then check for special variables (e.g.,
# __version__), and then return -- there's nothing else to do.
if (api_doc.docstring in (None, UNKNOWN)):
if isinstance(api_doc, NamespaceDoc):
for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
add_metadata_from_var(api_doc, field)
return
# Remove leading indentation from the docstring.
api_doc.docstring = unindent_docstring(api_doc.docstring)
# Decide which docformat is used by this module.
docformat = get_docformat(api_doc, docindex)
# A list of markup errors from parsing.
parse_errors = []
# Extract a signature from the docstring, if it has one. This
# overrides any signature we got via introspection/parsing.
if isinstance(api_doc, RoutineDoc):
parse_function_signature(api_doc, None, docformat, parse_errors)
# Parse the docstring. Any errors encountered are stored as
# `ParseError` objects in the errors list.
parsed_docstring = markup.parse(api_doc.docstring, docformat, parse_errors)
# Divide the docstring into a description and a list of
# fields.
descr, fields = parsed_docstring.split_fields(parse_errors)
api_doc.descr = descr
field_warnings = []
# Handle the constructor fields that have been defined in the class
# docstring. This code assumes that a class docstring is parsed before
# the same class __init__ docstring.
if isinstance(api_doc, ClassDoc):
# Parse ahead the __init__ docstring for this class
initvar = api_doc.variables.get('__init__')
if initvar and isinstance(initvar.value, RoutineDoc):
init_api_doc = initvar.value
parse_docstring(init_api_doc, docindex, suppress_warnings)
parse_function_signature(init_api_doc, api_doc, docformat,
parse_errors)
init_fields = split_init_fields(fields, field_warnings)
# Process fields
for field in init_fields:
try:
process_field(init_api_doc, docindex, field.tag(),
field.arg(), field.body())
except ValueError as e:
field_warnings.append(str(e))
# Process fields
for field in fields:
try:
process_field(api_doc, docindex, field.tag(), field.arg(),
field.body())
except ValueError as e:
field_warnings.append(str(e))
# Check to make sure that all type parameters correspond to
# some documented parameter.
check_type_fields(api_doc, field_warnings)
# Check for special variables (e.g., __version__)
if isinstance(api_doc, NamespaceDoc):
for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
add_metadata_from_var(api_doc, field)
# Extract a summary
if api_doc.summary is None and api_doc.descr is not None:
api_doc.summary, api_doc.other_docs = api_doc.descr.summary()
# If the summary is empty, but the return field is not, then use
# the return field to generate a summary description.
if (isinstance(api_doc, RoutineDoc) and api_doc.summary is None
and api_doc.return_descr is not None):
s, o = api_doc.return_descr.summary()
api_doc.summary = RETURN_PDS + s
api_doc.other_docs = o
# [XX] Make sure we don't have types/param descrs for unknown
# vars/params?
# Report any errors that occured
if api_doc in suppress_warnings:
if parse_errors or field_warnings:
log.info("Suppressing docstring warnings for %s, since it "
"is not included in the documented set." %
api_doc.canonical_name)
else:
report_errors(api_doc, docindex, parse_errors, field_warnings)
def parse_function_signature(func_doc, doc_source, docformat, parse_errors):
"""
Construct the signature for a builtin function or method from
its docstring. If the docstring uses the standard convention
of including a signature in the first line of the docstring
(and formats that signature according to standard
conventions), then it will be used to extract a signature.
Otherwise, the signature will be set to a single varargs
variable named C{"..."}.
@param func_doc: The target object where to store parsed signature. Also
container of the docstring to parse if doc_source is C{None}
@type func_doc: L{RoutineDoc}
@param doc_source: Contains the docstring to parse. If C{None}, parse
L{func_doc} docstring instead
@type doc_source: L{APIDoc}
@rtype: C{None}
"""
if doc_source is None:
doc_source = func_doc
# If there's no docstring, then don't do anything.
if not doc_source.docstring: return False
m = _SIGNATURE_RE.match(doc_source.docstring)
if m is None: return False
# Do I want to be this strict?
# Notice that __init__ must match the class name instead, if the signature
# comes from the class docstring
# if not (m.group('func') == func_doc.canonical_name[-1] or
# '_'+m.group('func') == func_doc.canonical_name[-1]):
# log.warning("Not extracting function signature from %s's "
# "docstring, since the name doesn't match." %
# func_doc.canonical_name)
# return False
params = m.group('params')
rtype = m.group('return')
selfparam = m.group('self')
# Extract the parameters from the signature.
func_doc.posargs = []
func_doc.vararg = None
func_doc.kwarg = None
if func_doc.posarg_defaults is UNKNOWN:
func_doc.posarg_defaults = []
if params:
# Figure out which parameters are optional.
while '[' in params or ']' in params:
m2 = re.match(r'(.*)\[([^\[\]]+)\](.*)', params)
if not m2: return False
(start, mid, end) = m2.groups()
mid = re.sub(r'((,|^)\s*[\w\-\.]+)', r'\1=...', mid)
params = start + mid + end
params = re.sub(r'=...=', r'=', params)
for name in params.split(','):
if '=' in name:
(name, default_repr) = name.split('=', 1)
default = GenericValueDoc(parse_repr=default_repr)
else:
default = None
name = name.strip()
if name == '...':
func_doc.vararg = '...'
elif name.startswith('**'):
func_doc.kwarg = name[2:]
elif name.startswith('*'):
func_doc.vararg = name[1:]
else:
func_doc.posargs.append(name)
if len(func_doc.posarg_defaults) < len(func_doc.posargs):
func_doc.posarg_defaults.append(default)
elif default is not None:
argnum = len(func_doc.posargs) - 1
func_doc.posarg_defaults[argnum] = default
# Extract the return type/value from the signature
if rtype:
func_doc.return_type = markup.parse(rtype,
docformat,
parse_errors,
inline=True)
# Add the self parameter, if it was specified.
if selfparam:
func_doc.posargs.insert(0, selfparam)
func_doc.posarg_defaults.insert(0, None)
# Remove the signature from the docstring.
doc_source.docstring = doc_source.docstring[m.end():]
# We found a signature.
return True
log.docstring_warning(message)
elif len(linenums) == 1:
log.docstring_warning("Line %s: %s" % (linenums[0], message))
else:
linenums = ', '.join(['%s' % l for l in linenums])
log.docstring_warning("Lines %s: %s" % (linenums, message))
# Display all field warnings.
for warning in field_warnings:
log.docstring_warning(warning)
# End the message block.
log.end_block()
RETURN_PDS = markup.parse('Returns:', markup='epytext')
"""A ParsedDocstring containing the text 'Returns'. This is used to
construct summary descriptions for routines that have empty C{descr},
but non-empty C{return_descr}."""
RETURN_PDS._tree.children[0].attribs['inline'] = True
######################################################################
#{ Field Processing Error Messages
######################################################################
UNEXPECTED_ARG = '%r did not expect an argument'
EXPECTED_ARG = '%r expected an argument'
EXPECTED_SINGLE_ARG = '%r expected a single argument'
BAD_CONTEXT = 'Invalid context for %r'
REDEFINED = 'Redefinition of %s'
UNKNOWN_TAG = 'Unknown field tag %r'
def parse_docstring(api_doc, docindex, suppress_warnings=[]):
"""
Process the given C{APIDoc}'s docstring. In particular, populate
the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
information provided by fields in the docstring.
@param docindex: A DocIndex, used to find the containing
module (to look up the docformat); and to find any
user docfields defined by containing objects.
@param suppress_warnings: A set of objects for which docstring
warnings should be suppressed.
"""
if api_doc.metadata is not UNKNOWN:
if not (isinstance(api_doc, RoutineDoc)
and api_doc.canonical_name[-1] == '__init__'):
log.debug("%s's docstring processed twice" %
api_doc.canonical_name)
return
initialize_api_doc(api_doc)
# If there's no docstring, then check for special variables (e.g.,
# __version__), and then return -- there's nothing else to do.
if (api_doc.docstring in (None, UNKNOWN)):
if isinstance(api_doc, NamespaceDoc):
for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
add_metadata_from_var(api_doc, field)
return
# Remove leading indentation from the docstring.
api_doc.docstring = unindent_docstring(api_doc.docstring)
# Decide which docformat is used by this module.
docformat = get_docformat(api_doc, docindex)
# A list of markup errors from parsing.
parse_errors = []
# Extract a signature from the docstring, if it has one. This
# overrides any signature we got via introspection/parsing.
if isinstance(api_doc, RoutineDoc):
parse_function_signature(api_doc, None, docformat, parse_errors)
# Parse the docstring. Any errors encountered are stored as
# `ParseError` objects in the errors list.
parsed_docstring = markup.parse(api_doc.docstring, docformat,
parse_errors)
# Divide the docstring into a description and a list of
# fields.
descr, fields = parsed_docstring.split_fields(parse_errors)
api_doc.descr = descr
field_warnings = []
# Handle the constructor fields that have been defined in the class
# docstring. This code assumes that a class docstring is parsed before
# the same class __init__ docstring.
if isinstance(api_doc, ClassDoc):
# Parse ahead the __init__ docstring for this class
initvar = api_doc.variables.get('__init__')
if initvar and isinstance(initvar.value, RoutineDoc):
init_api_doc = initvar.value
parse_docstring(init_api_doc, docindex, suppress_warnings)
parse_function_signature(init_api_doc, api_doc,
docformat, parse_errors)
init_fields = split_init_fields(fields, field_warnings)
# Process fields
for field in init_fields:
try:
process_field(init_api_doc, docindex, field.tag(),
field.arg(), field.body())
except ValueError, e: field_warnings.append(str(e))
# Process fields
for field in fields:
try:
process_field(api_doc, docindex, field.tag(),
field.arg(), field.body())
except ValueError, e: field_warnings.append(str(e))
# Check to make sure that all type parameters correspond to
# some documented parameter.
check_type_fields(api_doc, field_warnings)
# Check for special variables (e.g., __version__)
if isinstance(api_doc, NamespaceDoc):
for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
add_metadata_from_var(api_doc, field)
# Extract a summary
if api_doc.summary is None and api_doc.descr is not None:
api_doc.summary, api_doc.other_docs = api_doc.descr.summary()
# If the summary is empty, but the return field is not, then use
# the return field to generate a summary description.
if (isinstance(api_doc, RoutineDoc) and api_doc.summary is None and
api_doc.return_descr is not None):
s, o = api_doc.return_descr.summary()
api_doc.summary = RETURN_PDS + s
api_doc.other_docs = o
# [XX] Make sure we don't have types/param descrs for unknown
# vars/params?
# Report any errors that occured
if api_doc in suppress_warnings:
if parse_errors or field_warnings:
log.info("Suppressing docstring warnings for %s, since it "
"is not included in the documented set." %
api_doc.canonical_name)
else:
report_errors(api_doc, docindex, parse_errors, field_warnings)
def parse_function_signature(func_doc, doc_source, docformat, parse_errors):
"""
Construct the signature for a builtin function or method from
its docstring. If the docstring uses the standard convention
of including a signature in the first line of the docstring
(and formats that signature according to standard
conventions), then it will be used to extract a signature.
Otherwise, the signature will be set to a single varargs
variable named C{"..."}.
@param func_doc: The target object where to store parsed signature. Also
container of the docstring to parse if doc_source is C{None}
@type func_doc: L{RoutineDoc}
@param doc_source: Contains the docstring to parse. If C{None}, parse
L{func_doc} docstring instead
@type doc_source: L{APIDoc}
@rtype: C{None}
"""
if doc_source is None:
doc_source = func_doc
# If there's no docstring, then don't do anything.
if not doc_source.docstring: return False
m = _SIGNATURE_RE.match(doc_source.docstring)
if m is None: return False
# Do I want to be this strict?
# Notice that __init__ must match the class name instead, if the signature
# comes from the class docstring
# if not (m.group('func') == func_doc.canonical_name[-1] or
# '_'+m.group('func') == func_doc.canonical_name[-1]):
# log.warning("Not extracting function signature from %s's "
# "docstring, since the name doesn't match." %
# func_doc.canonical_name)
# return False
params = m.group('params')
rtype = m.group('return')
selfparam = m.group('self')
# Extract the parameters from the signature.
func_doc.posargs = []
func_doc.vararg = None
func_doc.kwarg = None
if func_doc.posarg_defaults is UNKNOWN:
func_doc.posarg_defaults = []
if params:
# Figure out which parameters are optional.
while '[' in params or ']' in params:
m2 = re.match(r'(.*)\[([^\[\]]+)\](.*)', params)
if not m2: return False
(start, mid, end) = m2.groups()
mid = re.sub(r'((,|^)\s*[\w\-\.]+)', r'\1=...', mid)
params = start+mid+end
params = re.sub(r'=...=' , r'=', params)
for name in params.split(','):
if '=' in name:
(name, default_repr) = name.split('=',1)
default = GenericValueDoc(parse_repr=default_repr)
else:
default = None
name = name.strip()
if name == '...':
func_doc.vararg = '...'
elif name.startswith('**'):
func_doc.kwarg = name[2:]
elif name.startswith('*'):
func_doc.vararg = name[1:]
else:
func_doc.posargs.append(name)
if len(func_doc.posarg_defaults) < len(func_doc.posargs):
func_doc.posarg_defaults.append(default)
elif default is not None:
argnum = len(func_doc.posargs)-1
func_doc.posarg_defaults[argnum] = default
# Extract the return type/value from the signature
if rtype:
func_doc.return_type = markup.parse(rtype, docformat, parse_errors,
inline=True)
# Add the self parameter, if it was specified.
if selfparam:
func_doc.posargs.insert(0, selfparam)
func_doc.posarg_defaults.insert(0, None)
# Remove the signature from the docstring.
doc_source.docstring = doc_source.docstring[m.end():]
# We found a signature.
return True
if len(linenums) == 0:
log.docstring_warning(message)
elif len(linenums) == 1:
log.docstring_warning("Line %s: %s" % (linenums[0], message))
else:
linenums = ', '.join(['%s' % l for l in linenums])
log.docstring_warning("Lines %s: %s" % (linenums, message))
# Display all field warnings.
for warning in field_warnings:
log.docstring_warning(warning)
# End the message block.
log.end_block()
RETURN_PDS = markup.parse('Returns:', markup='epytext')
"""A ParsedDocstring containing the text 'Returns'. This is used to
construct summary descriptions for routines that have empty C{descr},
but non-empty C{return_descr}."""
RETURN_PDS._tree.children[0].attribs['inline'] = True
######################################################################
#{ Field Processing Error Messages
######################################################################
UNEXPECTED_ARG = '%r did not expect an argument'
EXPECTED_ARG = '%r expected an argument'
EXPECTED_SINGLE_ARG = '%r expected a single argument'
BAD_CONTEXT = 'Invalid context for %r'
REDEFINED = 'Redefinition of %s'
UNKNOWN_TAG = 'Unknown field tag %r'
def parse_docstring(api_doc, docindex):
"""
Process the given C{APIDoc}'s docstring. In particular, populate
the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
information provided by fields in the docstring.
@param docindex: A DocIndex, used to find the containing
module (to look up the docformat); and to find any
user docfields defined by containing objects.
"""
if api_doc.metadata is not UNKNOWN:
if not (isinstance(api_doc, RoutineDoc)
and api_doc.canonical_name[-1] == '__init__'):
log.debug("%s's docstring processed twice" %
api_doc.canonical_name)
return
initialize_api_doc(api_doc)
# If there's no docstring, then there's nothing more to do.
if (api_doc.docstring in (None, UNKNOWN)):
return
# Remove leading indentation from the docstring.
api_doc.docstring = unindent_docstring(api_doc.docstring)
# Extract a signature from the docstring, if it has one. This
# overrides any signature we got via introspection/parsing.
if isinstance(api_doc, RoutineDoc):
parse_function_signature(api_doc)
# Parse the docstring. Any errors encountered are stored as
# `ParseError` objects in the errors list.
docformat = get_docformat(api_doc, docindex)
parse_errors = []
parsed_docstring = markup.parse(api_doc.docstring, docformat, parse_errors)
# Divide the docstring into a description and a list of
# fields.
descr, fields = parsed_docstring.split_fields(parse_errors)
api_doc.descr = descr
field_warnings = []
# Handle the constructor fields that have been defined in the class
# docstring. This code assumes that a class docstring is parsed before
# the same class __init__ docstring.
if isinstance(api_doc, ClassDoc):
# Parse ahead the __init__ docstring for this class
initvar = api_doc.variables.get('__init__')
if initvar and initvar.value not in (None, UNKNOWN):
init_api_doc = initvar.value
parse_docstring(init_api_doc, docindex)
parse_function_signature(init_api_doc, api_doc)
init_fields = split_init_fields(fields, field_warnings)
# Process fields
for field in init_fields:
try:
process_field(init_api_doc, docindex, field.tag(),
field.arg(), field.body())
except ValueError, e:
field_warnings.append(str(e))