def test_generate_empty_functional():
"""
Functional test for index generation of empty API.
Header is plain text while content is compressed.
"""
project_name = 'some-name'
log = []
logger = lambda part, message: log.append((part, message))
sut = SphinxInventory(logger=logger, project_name=project_name)
output = PersistentStringIO()
sut._openFileForWriting = lambda path: closing(output)
sut.generate(subjects=[], basepath='base-path')
expected_log = [(
'sphinx',
'Generating objects inventory at base-path/objects.inv'
)]
assert expected_log == log
expected_ouput = """# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
x\x9c\x03\x00\x00\x00\x00\x01"""
assert expected_ouput == output.getvalue()
def __init__(self, options: Optional[Values] = None):
self.allobjects: Dict[str, Documentable] = {}
self.rootobjects: List[Documentable] = []
self.violations = 0
"""The number of docstring problems found.
This is used to determine whether to fail the build when using
the --warnings-as-errors option, so it should only be increased
for problems that the user can fix.
"""
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.projectname = 'my project'
self.docstring_syntax_errors: Set[str] = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs: Set[Tuple[str, str]] = set()
self.unprocessed_modules: Set[Module] = set()
self.module_count = 0
self.processing_modules: List[str] = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def test_generate_empty_functional():
"""
Functional test for index generation of empty API.
Header is plain text while content is compressed.
"""
project_name = 'some-name'
log = []
logger = lambda section, message, thresh=0: log.append((
section, message, thresh))
sut = SphinxInventory(logger=logger, project_name=project_name)
output = PersistentStringIO()
sut._openFileForWriting = lambda path: closing(output)
sut.generate(subjects=[], basepath='base-path')
expected_log = [(
'sphinx',
'Generating objects inventory at base-path/objects.inv',
0
)]
assert expected_log == log
expected_ouput = """# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
x\x9c\x03\x00\x00\x00\x00\x01"""
assert expected_ouput == output.getvalue()
def __init__(self, options=None):
self.allobjects = OrderedDict()
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.docstring_syntax_errors = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def test_EpydocLinker_resolve_identifier_xref_intersphinx_relative_id(
) -> None:
"""
Return the link from inventory using short names, by resolving them based
on the imports done in the module.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['ext_package.ext_module'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
# Here we set up the target module as it would have this import.
# from ext_package import ext_module
ext_package = model.Module(system, 'ext_package')
target.contents['ext_module'] = model.Module(system,
'ext_module',
parent=ext_package)
sut = epydoc2stan._EpydocLinker(target)
# This is called for the L{ext_module<Pretty Text>} markup.
url = sut.resolve_identifier('ext_module')
url_xref = sut.resolve_identifier_xref('ext_module', 0)
assert "http://tm.tld/some.html" == url
assert "http://tm.tld/some.html" == url_xref
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def test_EpydocLinker_translate_identifier_xref_intersphinx_relative_id():
"""
Return the link from inventory using short names, by resolving them based
on the imports done in the module.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["ext_package.ext_module"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
# Here we set up the target module as it would have this import.
# from ext_package import ext_module
ext_package = model.Module(system, "ext_package", "ignore-docstring")
target.contents["ext_module"] = model.Module(system,
"ext_module",
"ignore-docstring",
parent=ext_package)
sut = epydoc2stan._EpydocLinker(target)
# This is called for the L{ext_module<Pretty Text>} markup.
result = sut.translate_identifier_xref("ext_module", "Pretty Text")
expected = '<a href="http://tm.tld/some.html"><code>Pretty Text</code></a>'
assert expected == result
def test_parseInventory_empty(
inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return empty dict for empty input.
"""
result = inv_reader_nolog._parseInventory('http://base.tld', '')
assert {} == result
def test_getLink_found(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return the link from internal state.
"""
inv_reader_nolog._links['some.name'] = ('http://base.tld', 'some/url.php')
assert 'http://base.tld/some/url.php' == inv_reader_nolog.getLink(
'some.name')
def test_parseInventory_single_line(
inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return a dict with a single member.
"""
result = inv_reader_nolog._parseInventory(
'http://base.tld', 'some.attr py:attr -1 some.html De scription')
assert {'some.attr': ('http://base.tld', 'some.html')} == result
def test_getLink_self_anchor(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return the link with anchor as target name when link end with $.
"""
inv_reader_nolog._links['some.name'] = ('http://base.tld',
'some/url.php#$')
assert 'http://base.tld/some/url.php#some.name' == inv_reader_nolog.getLink(
'some.name')
def test_initialization():
"""
Is initialized with logger and project name.
"""
logger = object()
name = object()
sut = SphinxInventory(logger=logger, project_name=name)
assert logger is sut.info
assert name is sut.project_name
def __setstate__(self, state):
if "abbrevmapping" not in state:
state["abbrevmapping"] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError(name)
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if "allobjects" not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith("$"):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith("@"):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith("!"):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(
logger=self.msg, project_name=self.projectname
)
def test_update_functional(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Functional test for updating from an empty inventory.
"""
payload = (b'some.module1 py:module -1 module1.html -\n'
b'other.module2 py:module 0 module2.html Other description\n')
# Patch URL loader to avoid hitting the system.
content = b"""# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
""" + zlib.compress(payload)
url = 'http://some.url/api/objects.inv'
inv_reader_nolog.update({url: content}, url)
assert 'http://some.url/api/module1.html' == inv_reader_nolog.getLink(
'some.module1')
assert 'http://some.url/api/module2.html' == inv_reader_nolog.getLink(
'other.module2')
def test_EpydocLinker_look_for_intersphinx_hit():
"""
Return the link from inventory based on first package name.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["base.module.other"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
sut = epydoc2stan._EpydocLinker(target)
result = sut.look_for_intersphinx("base.module.other")
assert "http://tm.tld/some.html" == result
def test_EpydocLinker_look_for_intersphinx_hit() -> None:
"""
Return the link from inventory based on first package name.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
sut = epydoc2stan._EpydocLinker(target)
result = sut.look_for_intersphinx('base.module.other')
assert 'http://tm.tld/some.html' == result
def test_getPayload_empty(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return empty string.
"""
content = b"""# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
x\x9c\x03\x00\x00\x00\x00\x01"""
result = inv_reader_nolog._getPayload('http://base.ignore', content)
assert '' == result
def test_getPayload_content(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return content as string.
"""
payload = "first_line\nsecond line\nit's a snake: \U0001F40D"
content = b"""# Ignored line
# Project: some-name
# Version: 2.0
# commented line.
""" + zlib.compress(payload.encode('utf-8'))
result = inv_reader_nolog._getPayload('http://base.ignore', content)
assert payload == result
def test_EpydocLinker_resolve_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
sut = epydoc2stan._EpydocLinker(target)
url = sut.resolve_identifier_xref('base.module.other')
assert "http://tm.tld/some.html" == url
def test_xref_link_intersphinx() -> None:
"""A linked name that is documented in another project is linked using
an absolute URL (retrieved via Intersphinx).
"""
mod = fromText('''
def func():
"""This is a thin wrapper around L{external.func}."""
''', modname='test')
system = mod.system
inventory = SphinxInventory(system.msg)
inventory._links['external.func'] = ('https://example.net', 'lib.html#func')
system.intersphinx = inventory
html = docstring2html(mod.contents['func'])
assert 'href="https://example.net/lib.html#func"' in html
def test_EpydocLinker_translate_identifier_xref_intersphinx():
"""
Return the link from inventory.
"""
system = model.System()
inventory = SphinxInventory(system.msg, 'some-project')
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name', 'ignore-docstring')
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref(
'base.module.other', 'base.module.pretty')
expected = (
'<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
)
assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["base.module.other"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref("base.module.other",
"base.module.pretty")
expected = '<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name', 'ignore-docstring')
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref('base.module.other',
'base.module.pretty')
expected = (
'<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
)
assert expected == flatten(result)
def __setstate__(self, state):
if 'abbrevmapping' not in state:
state['abbrevmapping'] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError, name
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if 'allobjects' not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith('$'):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith('@'):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith('!'):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
class System(object):
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# not done here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section, )
delta = max(
[self.options.verbosity_details.get(sect, 0) for sect in section])
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = '%s/%s' % (i, n)
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + i, msg, end='')
sys.stdout.flush()
if i == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section,
msg,
thresh=0,
topthresh=100,
nonl=False,
wantsnl=True,
once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print()
def objForFullName(self, fullName):
for system in [self] + self.moresystems:
if fullName in system.allobjects:
return system.allobjects[fullName]
return None
def _warning(self, current, type, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, type, detail)
self.warnings.setdefault(type, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.orderedallobjects:
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def __getstate__(self):
d = self.__dict__.copy()
del d['intersphinx']
return d
def __setstate__(self, state):
if 'abbrevmapping' not in state:
state['abbrevmapping'] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError(name)
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if 'allobjects' not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith('$'):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith('@'):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith('!'):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def addObject(self, obj):
"""Add C{object} to the system."""
if obj.parent:
obj.parent.orderedcontents.append(obj)
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
self.orderedallobjects.append(obj)
if obj.fullName() in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[obj.fullName()] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
return
projBaseDir = mod.system.options.projectbasedirectory
if projBaseDir is not None:
mod.sourceHref = (self.sourcebase +
mod.filepath[len(projBaseDir):])
return
trailing = []
dir, fname = os.path.split(mod.filepath)
while os.path.exists(os.path.join(dir, '.svn')):
dir, dirname = os.path.split(dir)
trailing.append(dirname)
# now trailing[-1] would be 'Divmod' in the above example
del trailing[-1]
trailing.reverse()
trailing.append(fname)
mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, None, parentPackage)
self.addObject(mod)
self.progress("addModule", len(self.orderedallobjects), None,
"modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, None, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, None, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.iteritems():
if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))):
f = self.Function(self, k, v.__doc__, parent)
f.parentMod = parentMod
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, v.__doc__, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
print(py_mod)
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception("package path %r does not exist!" % (dirpath, ))
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in os.listdir(dirpath):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, type) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if type == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = "%s.%s" % (package.fullName(),
module_name)
else:
module_full_name = module_name
py_mod = imp.load_module(module_full_name, open(path, 'rb'),
path, (suffix, mode, type))
self.introspectModule(py_mod, module_full_name)
elif type == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fn = obj.fullName()
while (fn + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[obj.fullName()]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
for c in o.orderedcontents:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.orderedcontents:
readd(c)
readd(prev)
self.allobjects[obj.fullName()] = obj
return obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state == UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state == UNPROCESSED
mod.state = PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress(
'process', self.module_count - len(self.unprocessed_modules),
self.module_count, "modules processed %s warnings" %
(sum(len(v) for v in self.warnings.itervalues()), ))
def process(self):
while self.unprocessed_modules:
mod = iter(self.unprocessed_modules).next()
self.processModule(mod)
def fetchIntersphinxInventories(self):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(url)
def test_getLink_not_found(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Return None if link does not exists.
"""
assert None is inv_reader_nolog.getLink('no.such.name')
class System(object):
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# not done here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section,)
delta = max([self.options.verbosity_details.get(sect, 0) for sect in section])
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = '%s/%s'%(i,n)
if self.verbosity(section) == 0 and sys.stdout.isatty():
print '\r'+i, msg,
sys.stdout.flush()
if i == n:
self.needsnl = False
print
else:
self.needsnl = True
def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print
print msg,
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print
def objForFullName(self, fullName):
for system in [self] + self.moresystems:
if fullName in system.allobjects:
return system.allobjects[fullName]
return None
def _warning(self, current, type, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print fn, type, detail
self.warnings.setdefault(type, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.orderedallobjects:
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def __getstate__(self):
d = self.__dict__.copy()
del d['intersphinx']
return d
def __setstate__(self, state):
if 'abbrevmapping' not in state:
state['abbrevmapping'] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError, name
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if 'allobjects' not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith('$'):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith('@'):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith('!'):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def addObject(self, obj):
"""Add C{object} to the system."""
if obj.parent:
obj.parent.orderedcontents.append(obj)
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
self.orderedallobjects.append(obj)
if obj.fullName() in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[obj.fullName()] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
return
projBaseDir = mod.system.options.projectbasedirectory
if projBaseDir is not None:
mod.sourceHref = (
self.sourcebase +
mod.filepath[len(projBaseDir):])
return
trailing = []
dir, fname = os.path.split(mod.filepath)
while os.path.exists(os.path.join(dir, '.svn')):
dir, dirname = os.path.split(dir)
trailing.append(dirname)
# now trailing[-1] would be 'Divmod' in the above example
del trailing[-1]
trailing.reverse()
trailing.append(fname)
mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, None, parentPackage)
self.addObject(mod)
self.progress(
"addModule", len(self.orderedallobjects),
None, "modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, None, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, None, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.iteritems():
if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))):
f = self.Function(self, k, v.__doc__, parent)
f.parentMod = parentMod
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, v.__doc__, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
print py_mod
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception("package path %r does not exist!"
%(dirpath,))
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in os.listdir(dirpath):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, type) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if type == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = "%s.%s" % (
package.fullName(), module_name)
else:
module_full_name = module_name
py_mod = imp.load_module(
module_full_name, open(path, 'rb'), path,
(suffix, mode, type))
self.introspectModule(py_mod, module_full_name)
elif type == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fn = obj.fullName()
while (fn + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[obj.fullName()]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
for c in o.orderedcontents:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.orderedcontents:
readd(c)
readd(prev)
self.allobjects[obj.fullName()] = obj
return obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state == UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state == UNPROCESSED
mod.state = PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule", "processing %s"%(self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress(
'process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
"modules processed %s warnings"%(
sum(len(v) for v in self.warnings.itervalues()),))
def process(self):
while self.unprocessed_modules:
mod = iter(self.unprocessed_modules).next()
self.processModule(mod)
def fetchIntersphinxInventories(self):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(url)
def make_SphinxInventory(logger=object()):
"""
Return a SphinxInventory.
"""
return SphinxInventory(logger=logger, project_name='project_name')
def main(args):
import cPickle
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
if options.inputpickle:
system = cPickle.load(open(options.inputpickle, 'rb'))
if options.systemclass:
if type(system) is not systemclass:
cls = type(system)
msg = ("loaded pickle has class %s.%s, differing "
"from explicitly requested %s")
error(msg, cls.__module__, cls.__name__,
options.systemclass)
else:
system = systemclass()
# Once pickle support is removed, always instantiate System with
# options and make fetchIntersphinxInventories private in __init__.
system.options = options
system.fetchIntersphinxInventories()
system.urlprefix = ''
if options.moresystems:
moresystems = []
for fnamepref in options.moresystems:
fname, prefix = fnamepref.split(':', 1)
moresystems.append(cPickle.load(open(fname, 'rb')))
moresystems[-1].urlprefix = prefix
moresystems[-1].options = system.options
moresystems[-1].subsystems.append(system)
system.moresystems = moresystems
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
if options.auto:
options.server = True
options.edit = True
for fn in os.listdir('.'):
if os.path.isdir(fn) and \
os.path.exists(os.path.join(fn, '__init__.py')):
options.packages.append(fn)
elif fn.endswith('.py') and fn != 'setup.py':
options.modules.append(fn)
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.outputpickle and not options.testing \
and not options.server and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError, e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(
system, m, None, prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', None, prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.livecheck:
error("write this")
if system.options.projectname is None:
name = '/'.join([ro.name for ro in system.rootobjects])
system.msg(
'warning',
'WARNING: guessing '+name+' for project name', thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: save the system, if desired
if options.outputpickle:
system.msg('', 'saving output pickle to ' + options.outputpickle)
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
# step 5: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(
options.htmlwriter, '--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg('html', 'writing html to %s using %s.%s'%(
options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
system.epytextproblems = []
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.epytextproblems:
def p(msg):
system.msg(('epytext', 'epytext-summary'), msg, thresh=-1, topthresh=1)
p("these %s objects' docstrings are not proper epytext:"
%(len(system.epytextproblems),))
exitcode = 2
for fn in system.epytextproblems:
p(' '+fn)
if options.outputpickle:
system.msg(
'', 'saving output pickle to ' + options.outputpickle)
# save again, with epytextproblems
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventory(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
# Finally, if we should serve html, lets serve some html.
if options.server:
from pydoctor.server import (
EditingPyDoctorResource, PyDoctorResource)
from pydoctor.epydoc2stan import doc2stan
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.web.vhost import VHostMonsterResource
from twisted.internet import reactor
if options.edit:
if not options.nocheck:
system.msg(
"server", "Checking formatting of docstrings.")
included_obs = [
ob for ob in system.orderedallobjects
if ob.isVisible]
for i, ob in enumerate(included_obs):
system.progress(
"server", i+1, len(included_obs),
"docstrings checked, found %s problems" % (
len(system.epytextproblems)))
doc2stan(ob, docstring=ob.docstring)
root = EditingPyDoctorResource(system)
else:
root = PyDoctorResource(system)
if options.facing_path:
options.local_only = True
realroot = Resource()
cur = realroot
realroot.putChild('vhost', VHostMonsterResource())
segments = options.facing_path.split('/')
for segment in segments[:-1]:
next = Resource()
cur.putChild(segment, next)
cur = next
cur.putChild(segments[-1], root)
root = realroot
system.msg(
"server",
"Setting up server at http://localhost:%d/" %
options.server_port)
if options.auto:
def wb_open():
import webbrowser
webbrowser.open(
'http://localhost:%d/' % options.server_port)
reactor.callWhenRunning(wb_open)
from twisted.python import log
log.startLogging(sys.stdout)
site = Site(root)
if options.local_only:
interface = 'localhost'
else:
interface = ''
reactor.listenTCP(options.server_port, site, interface=interface)
reactor.run()
class System:
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# Not assigned here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
defaultBuilder: Type[ASTBuilder]
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.docstring_syntax_errors = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section, )
delta = max(
self.options.verbosity_details.get(sect, 0) for sect in section)
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = f'{i}/{n}'
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + i, msg, end='')
sys.stdout.flush()
if i == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section,
msg,
thresh=0,
topthresh=100,
nonl=False,
wantsnl=True,
once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print('')
def objForFullName(self, fullName):
return self.allobjects.get(fullName)
def _warning(self, current, message, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, message, detail)
self.warnings.setdefault(message, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.allobjects.values():
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.kind is None:
return PrivacyClass.HIDDEN
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def addObject(self, obj):
"""Add C{object} to the system."""
fullName = obj.fullName()
if obj.parent and obj.parent.fullName() != fullName:
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
if fullName in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[fullName] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
else:
projBaseDir = mod.system.options.projectbasedirectory
mod.sourceHref = (self.sourcebase +
mod.filepath[len(projBaseDir):])
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, parentPackage)
self.addObject(mod)
self.progress("addModule", len(self.allobjects), None,
"modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.items():
if isinstance(v, (types.BuiltinFunctionType, types.FunctionType)):
f = self.Function(self, k, parent)
f.parentMod = parentMod
f.docstring = v.__doc__
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
c.docstring = v.__doc__
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception(f"package path {dirpath!r} does not exist!")
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in sorted(os.listdir(dirpath)):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, impl) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if impl == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = f'{package.fullName()}.{module_name}'
else:
module_full_name = module_name
py_mod = imp.load_module(module_full_name, open(path, 'rb'),
path, (suffix, mode, impl))
self.introspectModule(py_mod, module_full_name)
elif impl == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fullName = obj.fullName()
while (fullName + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[fullName]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
oc = list(o.contents.values())
for c in oc:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.contents.values():
readd(c)
readd(prev)
self.allobjects[fullName] = obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state is ProcessingState.UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state is ProcessingState.UNPROCESSED
mod.state = ProcessingState.PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = ProcessingState.PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
num_warnings = sum(len(v) for v in self.warnings.values())
self.progress('process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
f"modules processed {num_warnings} warnings")
def process(self):
while self.unprocessed_modules:
mod = next(iter(self.unprocessed_modules))
self.processModule(mod)
def fetchIntersphinxInventories(self, cache):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(cache, url)
def main(args):
import cPickle
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
if options.inputpickle:
system = cPickle.load(open(options.inputpickle, 'rb'))
if options.systemclass:
if type(system) is not systemclass:
cls = type(system)
msg = ("loaded pickle has class %s.%s, differing "
"from explicitly requested %s")
error(msg, cls.__module__, cls.__name__,
options.systemclass)
else:
system = systemclass()
# Once pickle support is removed, always instantiate System with
# options and make fetchIntersphinxInventories private in __init__.
system.options = options
system.fetchIntersphinxInventories(cache)
system.urlprefix = ''
if options.moresystems:
moresystems = []
for fnamepref in options.moresystems:
fname, prefix = fnamepref.split(':', 1)
moresystems.append(cPickle.load(open(fname, 'rb')))
moresystems[-1].urlprefix = prefix
moresystems[-1].options = system.options
moresystems[-1].subsystems.append(system)
system.moresystems = moresystems
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.outputpickle and not options.testing \
and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(e)
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(system, m, None,
prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', None,
prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.livecheck:
error("write this")
if system.options.projectname is None:
name = '/'.join([ro.name for ro in system.rootobjects])
system.msg('warning',
'WARNING: guessing ' + name + ' for project name',
thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: save the system, if desired
if options.outputpickle:
system.msg('', 'saving output pickle to ' + options.outputpickle)
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
# step 5: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(options.htmlwriter,
'--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg(
'html', 'writing html to %s using %s.%s' %
(options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
system.epytextproblems = []
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.epytextproblems:
def p(msg):
system.msg(('epytext', 'epytext-summary'),
msg,
thresh=-1,
topthresh=1)
p("these %s objects' docstrings are not proper epytext:" %
(len(system.epytextproblems), ))
exitcode = 2
for fn in system.epytextproblems:
p(' ' + fn)
if options.outputpickle:
system.msg('',
'saving output pickle to ' + options.outputpickle)
# save again, with epytextproblems
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventory(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_traceback)
raise
return exitcode
class System:
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# Not assigned here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
defaultBuilder: Type[ASTBuilder]
sourcebase: Optional[str] = None
def __init__(self, options: Optional[Values] = None):
self.allobjects: Dict[str, Documentable] = {}
self.rootobjects: List[Documentable] = []
self.violations = 0
"""The number of docstring problems found.
This is used to determine whether to fail the build when using
the --warnings-as-errors option, so it should only be increased
for problems that the user can fix.
"""
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.projectname = 'my project'
self.docstring_syntax_errors: Set[str] = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs: Set[Tuple[str, str]] = set()
self.unprocessed_modules: Set[Module] = set()
self.module_count = 0
self.processing_modules: List[str] = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
@property
def root_names(self) -> Collection[str]:
"""The top-level package/module names in this system."""
return {
obj.name
for obj in self.rootobjects if isinstance(obj, (Module, Package))
}
def verbosity(self, section: Union[str, Iterable[str]]) -> int:
if isinstance(section, str):
section = (section, )
delta: int = max(
self.options.verbosity_details.get(sect, 0) for sect in section)
base: int = self.options.verbosity
return base + delta
def progress(self, section: str, i: int, n: Optional[int],
msg: str) -> None:
if n is None:
d = str(i)
else:
d = f'{i}/{n}'
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + d, msg, end='')
sys.stdout.flush()
if d == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section: str,
msg: str,
thresh: int = 0,
topthresh: int = 100,
nonl: bool = False,
wantsnl: bool = True,
once: bool = False) -> None:
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh < 0:
# Apidoc build messages are generated using negative threshold
# and we have separate reporting for them,
# on top of the logging system.
self.violations += 1
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print('')
def objForFullName(self, fullName: str) -> Optional[Documentable]:
return self.allobjects.get(fullName)
def find_object(self, full_name: str) -> Optional[Documentable]:
"""Look up an object using a potentially outdated full name.
A name can become outdated if the object is reparented:
L{objForFullName()} will only be able to find it under its new name,
but we might still have references to the old name.
@param full_name: The fully qualified name of the object.
@return: The object, or L{None} if the name is external (it does not
match any of the roots of this system).
@raise LookupError: If the object is not found, while its name does
match one of the roots of this system.
"""
obj = self.objForFullName(full_name)
if obj is not None:
return obj
# The object might have been reparented, in which case there will
# be an alias at the original location; look for it using expandName().
name_parts = full_name.split('.', 1)
for root_obj in self.rootobjects:
if root_obj.name == name_parts[0]:
obj = self.objForFullName(root_obj.expandName(name_parts[1]))
if obj is not None:
return obj
raise LookupError(full_name)
return None
def _warning(self, current: Optional[Documentable], message: str,
detail: str) -> None:
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, message, detail)
def objectsOfType(self, cls: Type[T]) -> Iterator[T]:
"""Iterate over all instances of C{cls} present in the system. """
for o in self.allobjects.values():
if isinstance(o, cls):
yield o
def privacyClass(self, ob: Documentable) -> PrivacyClass:
if ob.kind is None:
return PrivacyClass.HIDDEN
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def addObject(self, obj: Documentable) -> None:
"""Add C{object} to the system."""
if obj.parent:
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
first = self.allobjects.setdefault(obj.fullName(), obj)
if obj is not first:
self.handleDuplicate(obj)
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod: _ModuleT, source_path: Path) -> None:
if self.sourcebase is None:
mod.sourceHref = None
else:
projBaseDir = mod.system.options.projectbasedirectory
relative = source_path.relative_to(projBaseDir).as_posix()
mod.sourceHref = f'{self.sourcebase}/{relative}'
def addModule(self,
modpath: Path,
modname: str,
parentPackage: Optional[_PackageT] = None) -> None:
mod = self.Module(self, modname, parentPackage, modpath)
self.addObject(mod)
self.progress("addModule", len(self.allobjects), None,
"modules and packages discovered")
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod, modpath)
def ensureModule(self, module_full_name: str, modpath: Path) -> _ModuleT:
try:
module = self.allobjects[module_full_name]
assert isinstance(module, Module)
except KeyError:
pass
else:
return module
parent_package: Optional[_PackageT]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, parent_package, modpath)
self.addObject(module)
return module
def ensurePackage(self, package_full_name: str) -> _PackageT:
if package_full_name in self.allobjects:
package = self.allobjects[package_full_name]
assert isinstance(package, Package)
return package
parent_package: Optional[_PackageT]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing: object, parent: Documentable,
parentMod: _ModuleT) -> None:
for k, v in thing.__dict__.items():
if (isinstance(v, (types.BuiltinFunctionType, types.FunctionType))
# In PyPy 7.3.1, functions from extensions are not
# instances of the above abstract types.
or v.__class__.__name__ == 'builtin_function_or_method'):
f = self.Function(self, k, parent)
f.parentMod = parentMod
f.docstring = v.__doc__
f.decorators = None
f.signature = Signature()
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
c.docstring = v.__doc__
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, path: Path, module_full_name: str) -> None:
spec = importlib.util.spec_from_file_location(module_full_name, path)
py_mod = importlib.util.module_from_spec(spec)
loader = spec.loader
assert isinstance(loader, importlib.abc.Loader), loader
loader.exec_module(py_mod)
module = self.ensureModule(module_full_name, path)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
def addPackage(self,
dirpath: str,
parentPackage: Optional[_PackageT] = None) -> None:
if not os.path.exists(dirpath):
raise Exception(f"package path {dirpath!r} does not exist!")
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
for fname in sorted(os.listdir(dirpath)):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package: Optional[_PackageT],
path: str) -> None:
for suffix in importlib.machinery.all_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if suffix in importlib.machinery.EXTENSION_SUFFIXES:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = f'{package.fullName()}.{module_name}'
else:
module_full_name = module_name
self.introspectModule(Path(path), module_full_name)
elif suffix in importlib.machinery.SOURCE_SUFFIXES:
self.addModule(Path(path), module_name, package)
break
def handleDuplicate(self, obj: Documentable) -> None:
'''This is called when we see two objects with the same
.fullName(), for example::
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fullName = obj.fullName()
while (fullName + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[fullName]
self._warning(obj.parent, "duplicate", str(prev))
def remove(o: Documentable) -> None:
del self.allobjects[o.fullName()]
oc = list(o.contents.values())
for c in oc:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o: Documentable) -> None:
self.allobjects[o.fullName()] = o
for c in o.contents.values():
readd(c)
readd(prev)
self.allobjects[fullName] = obj
def getProcessedModule(self, modname: str) -> Optional[_ModuleT]:
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__')
if not isinstance(mod, Module):
return None
if mod.state is ProcessingState.UNPROCESSED:
self.processModule(mod)
assert mod.state in (ProcessingState.PROCESSING,
ProcessingState.PROCESSED)
return mod
def processModule(self, mod: _ModuleT) -> None:
assert mod.state is ProcessingState.UNPROCESSED
mod.state = ProcessingState.PROCESSING
if mod.source_path is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.source_path)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = ProcessingState.PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress('process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
f"modules processed, {self.violations} warnings")
def process(self) -> None:
while self.unprocessed_modules:
mod = next(iter(self.unprocessed_modules))
self.processModule(mod)
self.postProcess()
def postProcess(self) -> None:
"""Called when there are no more unprocessed modules.
Analysis of relations between documentables can be done here,
without the risk of drawing incorrect conclusions because modules
were not fully processed yet.
"""
pass
def fetchIntersphinxInventories(self, cache: CacheT) -> None:
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(cache, url)