def test_update_functional(inv_reader_nolog: sphinx.SphinxInventory) -> None:
"""
Functional test for updating from an empty inventory.
"""
payload = (b'some.module1 py:module -1 module1.html -\n'
b'other.module2 py:module 0 module2.html Other description\n')
# Patch URL loader to avoid hitting the system.
content = b"""# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
""" + zlib.compress(payload)
url = 'http://some.url/api/objects.inv'
inv_reader_nolog.update({url: content}, url)
assert 'http://some.url/api/module1.html' == inv_reader_nolog.getLink(
'some.module1')
assert 'http://some.url/api/module2.html' == inv_reader_nolog.getLink(
'other.module2')
class System(object):
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# not done here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section, )
delta = max(
[self.options.verbosity_details.get(sect, 0) for sect in section])
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = '%s/%s' % (i, n)
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + i, msg, end='')
sys.stdout.flush()
if i == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section,
msg,
thresh=0,
topthresh=100,
nonl=False,
wantsnl=True,
once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print()
def objForFullName(self, fullName):
for system in [self] + self.moresystems:
if fullName in system.allobjects:
return system.allobjects[fullName]
return None
def _warning(self, current, type, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, type, detail)
self.warnings.setdefault(type, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.orderedallobjects:
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def __getstate__(self):
d = self.__dict__.copy()
del d['intersphinx']
return d
def __setstate__(self, state):
if 'abbrevmapping' not in state:
state['abbrevmapping'] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError(name)
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if 'allobjects' not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith('$'):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith('@'):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith('!'):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def addObject(self, obj):
"""Add C{object} to the system."""
if obj.parent:
obj.parent.orderedcontents.append(obj)
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
self.orderedallobjects.append(obj)
if obj.fullName() in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[obj.fullName()] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
return
projBaseDir = mod.system.options.projectbasedirectory
if projBaseDir is not None:
mod.sourceHref = (self.sourcebase +
mod.filepath[len(projBaseDir):])
return
trailing = []
dir, fname = os.path.split(mod.filepath)
while os.path.exists(os.path.join(dir, '.svn')):
dir, dirname = os.path.split(dir)
trailing.append(dirname)
# now trailing[-1] would be 'Divmod' in the above example
del trailing[-1]
trailing.reverse()
trailing.append(fname)
mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, None, parentPackage)
self.addObject(mod)
self.progress("addModule", len(self.orderedallobjects), None,
"modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, None, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, None, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.iteritems():
if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))):
f = self.Function(self, k, v.__doc__, parent)
f.parentMod = parentMod
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, v.__doc__, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
print(py_mod)
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception("package path %r does not exist!" % (dirpath, ))
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in os.listdir(dirpath):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, type) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if type == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = "%s.%s" % (package.fullName(),
module_name)
else:
module_full_name = module_name
py_mod = imp.load_module(module_full_name, open(path, 'rb'),
path, (suffix, mode, type))
self.introspectModule(py_mod, module_full_name)
elif type == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fn = obj.fullName()
while (fn + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[obj.fullName()]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
for c in o.orderedcontents:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.orderedcontents:
readd(c)
readd(prev)
self.allobjects[obj.fullName()] = obj
return obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state == UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state == UNPROCESSED
mod.state = PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress(
'process', self.module_count - len(self.unprocessed_modules),
self.module_count, "modules processed %s warnings" %
(sum(len(v) for v in self.warnings.itervalues()), ))
def process(self):
while self.unprocessed_modules:
mod = iter(self.unprocessed_modules).next()
self.processModule(mod)
def fetchIntersphinxInventories(self):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(url)
class System:
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# Not assigned here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
defaultBuilder: Type[ASTBuilder]
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.docstring_syntax_errors = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section, )
delta = max(
self.options.verbosity_details.get(sect, 0) for sect in section)
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = f'{i}/{n}'
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + i, msg, end='')
sys.stdout.flush()
if i == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section,
msg,
thresh=0,
topthresh=100,
nonl=False,
wantsnl=True,
once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print('')
def objForFullName(self, fullName):
return self.allobjects.get(fullName)
def _warning(self, current, message, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, message, detail)
self.warnings.setdefault(message, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.allobjects.values():
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.kind is None:
return PrivacyClass.HIDDEN
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def addObject(self, obj):
"""Add C{object} to the system."""
fullName = obj.fullName()
if obj.parent and obj.parent.fullName() != fullName:
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
if fullName in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[fullName] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
else:
projBaseDir = mod.system.options.projectbasedirectory
mod.sourceHref = (self.sourcebase +
mod.filepath[len(projBaseDir):])
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, parentPackage)
self.addObject(mod)
self.progress("addModule", len(self.allobjects), None,
"modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.items():
if isinstance(v, (types.BuiltinFunctionType, types.FunctionType)):
f = self.Function(self, k, parent)
f.parentMod = parentMod
f.docstring = v.__doc__
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
c.docstring = v.__doc__
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception(f"package path {dirpath!r} does not exist!")
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in sorted(os.listdir(dirpath)):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, impl) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if impl == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = f'{package.fullName()}.{module_name}'
else:
module_full_name = module_name
py_mod = imp.load_module(module_full_name, open(path, 'rb'),
path, (suffix, mode, impl))
self.introspectModule(py_mod, module_full_name)
elif impl == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fullName = obj.fullName()
while (fullName + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[fullName]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
oc = list(o.contents.values())
for c in oc:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.contents.values():
readd(c)
readd(prev)
self.allobjects[fullName] = obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state is ProcessingState.UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state is ProcessingState.UNPROCESSED
mod.state = ProcessingState.PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = ProcessingState.PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
num_warnings = sum(len(v) for v in self.warnings.values())
self.progress('process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
f"modules processed {num_warnings} warnings")
def process(self):
while self.unprocessed_modules:
mod = next(iter(self.unprocessed_modules))
self.processModule(mod)
def fetchIntersphinxInventories(self, cache):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(cache, url)
class System:
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# Not assigned here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
defaultBuilder: Type[ASTBuilder]
sourcebase: Optional[str] = None
def __init__(self, options: Optional[Values] = None):
self.allobjects: Dict[str, Documentable] = {}
self.rootobjects: List[Documentable] = []
self.violations = 0
"""The number of docstring problems found.
This is used to determine whether to fail the build when using
the --warnings-as-errors option, so it should only be increased
for problems that the user can fix.
"""
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.projectname = 'my project'
self.docstring_syntax_errors: Set[str] = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs: Set[Tuple[str, str]] = set()
self.unprocessed_modules: Set[Module] = set()
self.module_count = 0
self.processing_modules: List[str] = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
@property
def root_names(self) -> Collection[str]:
"""The top-level package/module names in this system."""
return {
obj.name
for obj in self.rootobjects if isinstance(obj, (Module, Package))
}
def verbosity(self, section: Union[str, Iterable[str]]) -> int:
if isinstance(section, str):
section = (section, )
delta: int = max(
self.options.verbosity_details.get(sect, 0) for sect in section)
base: int = self.options.verbosity
return base + delta
def progress(self, section: str, i: int, n: Optional[int],
msg: str) -> None:
if n is None:
d = str(i)
else:
d = f'{i}/{n}'
if self.verbosity(section) == 0 and sys.stdout.isatty():
print('\r' + d, msg, end='')
sys.stdout.flush()
if d == n:
self.needsnl = False
print()
else:
self.needsnl = True
def msg(self,
section: str,
msg: str,
thresh: int = 0,
topthresh: int = 100,
nonl: bool = False,
wantsnl: bool = True,
once: bool = False) -> None:
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh < 0:
# Apidoc build messages are generated using negative threshold
# and we have separate reporting for them,
# on top of the logging system.
self.violations += 1
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print()
print(msg, end='')
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print('')
def objForFullName(self, fullName: str) -> Optional[Documentable]:
return self.allobjects.get(fullName)
def find_object(self, full_name: str) -> Optional[Documentable]:
"""Look up an object using a potentially outdated full name.
A name can become outdated if the object is reparented:
L{objForFullName()} will only be able to find it under its new name,
but we might still have references to the old name.
@param full_name: The fully qualified name of the object.
@return: The object, or L{None} if the name is external (it does not
match any of the roots of this system).
@raise LookupError: If the object is not found, while its name does
match one of the roots of this system.
"""
obj = self.objForFullName(full_name)
if obj is not None:
return obj
# The object might have been reparented, in which case there will
# be an alias at the original location; look for it using expandName().
name_parts = full_name.split('.', 1)
for root_obj in self.rootobjects:
if root_obj.name == name_parts[0]:
obj = self.objForFullName(root_obj.expandName(name_parts[1]))
if obj is not None:
return obj
raise LookupError(full_name)
return None
def _warning(self, current: Optional[Documentable], message: str,
detail: str) -> None:
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print(fn, message, detail)
def objectsOfType(self, cls: Type[T]) -> Iterator[T]:
"""Iterate over all instances of C{cls} present in the system. """
for o in self.allobjects.values():
if isinstance(o, cls):
yield o
def privacyClass(self, ob: Documentable) -> PrivacyClass:
if ob.kind is None:
return PrivacyClass.HIDDEN
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def addObject(self, obj: Documentable) -> None:
"""Add C{object} to the system."""
if obj.parent:
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
first = self.allobjects.setdefault(obj.fullName(), obj)
if obj is not first:
self.handleDuplicate(obj)
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod: _ModuleT, source_path: Path) -> None:
if self.sourcebase is None:
mod.sourceHref = None
else:
projBaseDir = mod.system.options.projectbasedirectory
relative = source_path.relative_to(projBaseDir).as_posix()
mod.sourceHref = f'{self.sourcebase}/{relative}'
def addModule(self,
modpath: Path,
modname: str,
parentPackage: Optional[_PackageT] = None) -> None:
mod = self.Module(self, modname, parentPackage, modpath)
self.addObject(mod)
self.progress("addModule", len(self.allobjects), None,
"modules and packages discovered")
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod, modpath)
def ensureModule(self, module_full_name: str, modpath: Path) -> _ModuleT:
try:
module = self.allobjects[module_full_name]
assert isinstance(module, Module)
except KeyError:
pass
else:
return module
parent_package: Optional[_PackageT]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, parent_package, modpath)
self.addObject(module)
return module
def ensurePackage(self, package_full_name: str) -> _PackageT:
if package_full_name in self.allobjects:
package = self.allobjects[package_full_name]
assert isinstance(package, Package)
return package
parent_package: Optional[_PackageT]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing: object, parent: Documentable,
parentMod: _ModuleT) -> None:
for k, v in thing.__dict__.items():
if (isinstance(v, (types.BuiltinFunctionType, types.FunctionType))
# In PyPy 7.3.1, functions from extensions are not
# instances of the above abstract types.
or v.__class__.__name__ == 'builtin_function_or_method'):
f = self.Function(self, k, parent)
f.parentMod = parentMod
f.docstring = v.__doc__
f.decorators = None
f.signature = Signature()
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
c.docstring = v.__doc__
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, path: Path, module_full_name: str) -> None:
spec = importlib.util.spec_from_file_location(module_full_name, path)
py_mod = importlib.util.module_from_spec(spec)
loader = spec.loader
assert isinstance(loader, importlib.abc.Loader), loader
loader.exec_module(py_mod)
module = self.ensureModule(module_full_name, path)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
def addPackage(self,
dirpath: str,
parentPackage: Optional[_PackageT] = None) -> None:
if not os.path.exists(dirpath):
raise Exception(f"package path {dirpath!r} does not exist!")
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
for fname in sorted(os.listdir(dirpath)):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package: Optional[_PackageT],
path: str) -> None:
for suffix in importlib.machinery.all_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if suffix in importlib.machinery.EXTENSION_SUFFIXES:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = f'{package.fullName()}.{module_name}'
else:
module_full_name = module_name
self.introspectModule(Path(path), module_full_name)
elif suffix in importlib.machinery.SOURCE_SUFFIXES:
self.addModule(Path(path), module_name, package)
break
def handleDuplicate(self, obj: Documentable) -> None:
'''This is called when we see two objects with the same
.fullName(), for example::
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fullName = obj.fullName()
while (fullName + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[fullName]
self._warning(obj.parent, "duplicate", str(prev))
def remove(o: Documentable) -> None:
del self.allobjects[o.fullName()]
oc = list(o.contents.values())
for c in oc:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o: Documentable) -> None:
self.allobjects[o.fullName()] = o
for c in o.contents.values():
readd(c)
readd(prev)
self.allobjects[fullName] = obj
def getProcessedModule(self, modname: str) -> Optional[_ModuleT]:
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__')
if not isinstance(mod, Module):
return None
if mod.state is ProcessingState.UNPROCESSED:
self.processModule(mod)
assert mod.state in (ProcessingState.PROCESSING,
ProcessingState.PROCESSED)
return mod
def processModule(self, mod: _ModuleT) -> None:
assert mod.state is ProcessingState.UNPROCESSED
mod.state = ProcessingState.PROCESSING
if mod.source_path is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.source_path)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule",
"processing %s" % (self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = ProcessingState.PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress('process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
f"modules processed, {self.violations} warnings")
def process(self) -> None:
while self.unprocessed_modules:
mod = next(iter(self.unprocessed_modules))
self.processModule(mod)
self.postProcess()
def postProcess(self) -> None:
"""Called when there are no more unprocessed modules.
Analysis of relations between documentables can be done here,
without the risk of drawing incorrect conclusions because modules
were not fully processed yet.
"""
pass
def fetchIntersphinxInventories(self, cache: CacheT) -> None:
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(cache, url)
class System(object):
"""A collection of related documentable objects.
PyDoctor documents collections of objects, often the contents of a
package.
"""
Class = Class
Module = Module
Package = Package
Function = Function
Attribute = Attribute
# not done here for circularity reasons:
#defaultBuilder = astbuilder.ASTBuilder
sourcebase = None
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def verbosity(self, section=None):
if isinstance(section, str):
section = (section,)
delta = max([self.options.verbosity_details.get(sect, 0) for sect in section])
return self.options.verbosity + delta
def progress(self, section, i, n, msg):
if n is None:
i = str(i)
else:
i = '%s/%s'%(i,n)
if self.verbosity(section) == 0 and sys.stdout.isatty():
print '\r'+i, msg,
sys.stdout.flush()
if i == n:
self.needsnl = False
print
else:
self.needsnl = True
def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False):
if once:
if (section, msg) in self.once_msgs:
return
else:
self.once_msgs.add((section, msg))
if thresh <= self.verbosity(section) <= topthresh:
if self.needsnl and wantsnl:
print
print msg,
if nonl:
self.needsnl = True
sys.stdout.flush()
else:
self.needsnl = False
print
def objForFullName(self, fullName):
for system in [self] + self.moresystems:
if fullName in system.allobjects:
return system.allobjects[fullName]
return None
def _warning(self, current, type, detail):
if current is not None:
fn = current.fullName()
else:
fn = '<None>'
if self.options.verbosity > 0:
print fn, type, detail
self.warnings.setdefault(type, []).append((fn, detail))
def objectsOfType(self, cls):
"""Iterate over all instances of C{cls} present in the system. """
for o in self.orderedallobjects:
if isinstance(o, cls):
yield o
def privacyClass(self, ob):
if ob.name.startswith('_') and \
not (ob.name.startswith('__') and ob.name.endswith('__')):
return PrivacyClass.PRIVATE
return PrivacyClass.VISIBLE
def __getstate__(self):
d = self.__dict__.copy()
del d['intersphinx']
return d
def __setstate__(self, state):
if 'abbrevmapping' not in state:
state['abbrevmapping'] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError, name
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if 'allobjects' not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith('$'):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith('@'):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith('!'):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def addObject(self, obj):
"""Add C{object} to the system."""
if obj.parent:
obj.parent.orderedcontents.append(obj)
obj.parent.contents[obj.name] = obj
else:
self.rootobjects.append(obj)
self.orderedallobjects.append(obj)
if obj.fullName() in self.allobjects:
self.handleDuplicate(obj)
else:
self.allobjects[obj.fullName()] = obj
# if we assume:
#
# - that svn://divmod.org/trunk is checked out into ~/src/Divmod
#
# - that http://divmod.org/trac/browser/trunk is the trac URL to the
# above directory
#
# - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
# "--add-package" argument
#
# we want to work out the sourceHref for nevow.flat.ten. the answer
# is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
#
# we can work this out by finding that Divmod is the top of the svn
# checkout, and posixpath.join-ing the parts of the filePath that
# follows that.
#
# http://divmod.org/trac/browser/trunk
# ~/src/Divmod/Nevow/nevow/flat/ten.py
def setSourceHref(self, mod):
if self.sourcebase is None:
mod.sourceHref = None
return
projBaseDir = mod.system.options.projectbasedirectory
if projBaseDir is not None:
mod.sourceHref = (
self.sourcebase +
mod.filepath[len(projBaseDir):])
return
trailing = []
dir, fname = os.path.split(mod.filepath)
while os.path.exists(os.path.join(dir, '.svn')):
dir, dirname = os.path.split(dir)
trailing.append(dirname)
# now trailing[-1] would be 'Divmod' in the above example
del trailing[-1]
trailing.reverse()
trailing.append(fname)
mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)
def addModule(self, modpath, modname, parentPackage=None):
mod = self.Module(self, modname, None, parentPackage)
self.addObject(mod)
self.progress(
"addModule", len(self.orderedallobjects),
None, "modules and packages discovered")
mod.filepath = modpath
self.unprocessed_modules.add(mod)
self.module_count += 1
self.setSourceHref(mod)
def ensureModule(self, module_full_name):
if module_full_name in self.allobjects:
return self.allobjects[module_full_name]
if '.' in module_full_name:
parent_name, module_name = module_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
module_name = module_full_name
module = self.Module(self, module_name, None, parent_package)
self.addObject(module)
return module
def ensurePackage(self, package_full_name):
if package_full_name in self.allobjects:
return self.allobjects[package_full_name]
if '.' in package_full_name:
parent_name, package_name = package_full_name.rsplit('.', 1)
parent_package = self.ensurePackage(parent_name)
else:
parent_package = None
package_name = package_full_name
package = self.Package(self, package_name, None, parent_package)
self.addObject(package)
return package
def _introspectThing(self, thing, parent, parentMod):
for k, v in thing.__dict__.iteritems():
if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))):
f = self.Function(self, k, v.__doc__, parent)
f.parentMod = parentMod
f.decorators = None
f.argspec = ((), None, None, ())
self.addObject(f)
elif isinstance(v, type):
c = self.Class(self, k, v.__doc__, parent)
c.bases = []
c.baseobjects = []
c.rawbases = []
c.parentMod = parentMod
self.addObject(c)
self._introspectThing(v, c, parentMod)
def introspectModule(self, py_mod, module_full_name):
module = self.ensureModule(module_full_name)
module.docstring = py_mod.__doc__
self._introspectThing(py_mod, module, module)
print py_mod
def addPackage(self, dirpath, parentPackage=None):
if not os.path.exists(dirpath):
raise Exception("package path %r does not exist!"
%(dirpath,))
if not os.path.exists(os.path.join(dirpath, '__init__.py')):
raise Exception("you must pass a package directory to "
"addPackage")
if parentPackage:
prefix = parentPackage.fullName() + '.'
else:
prefix = ''
package_name = os.path.basename(dirpath)
package_full_name = prefix + package_name
package = self.ensurePackage(package_full_name)
package.filepath = dirpath
self.setSourceHref(package)
for fname in os.listdir(dirpath):
fullname = os.path.join(dirpath, fname)
if os.path.isdir(fullname):
initname = os.path.join(fullname, '__init__.py')
if os.path.exists(initname):
self.addPackage(fullname, package)
elif not fname.startswith('.'):
self.addModuleFromPath(package, fullname)
def addModuleFromPath(self, package, path):
for (suffix, mode, type) in imp.get_suffixes():
if not path.endswith(suffix):
continue
module_name = os.path.basename(path[:-len(suffix)])
if type == imp.C_EXTENSION:
if not self.options.introspect_c_modules:
continue
if package is not None:
module_full_name = "%s.%s" % (
package.fullName(), module_name)
else:
module_full_name = module_name
py_mod = imp.load_module(
module_full_name, open(path, 'rb'), path,
(suffix, mode, type))
self.introspectModule(py_mod, module_full_name)
elif type == imp.PY_SOURCE:
self.addModule(path, module_name, package)
break
def handleDuplicate(self, obj):
'''This is called when we see two objects with the same
.fullName(), for example:
class C:
if something:
def meth(self):
implementation 1
else:
def meth(self):
implementation 2
The default is that the second definition "wins".
'''
i = 0
fn = obj.fullName()
while (fn + ' ' + str(i)) in self.allobjects:
i += 1
prev = self.allobjects[obj.fullName()]
self._warning(obj.parent, "duplicate", prev)
def remove(o):
del self.allobjects[o.fullName()]
for c in o.orderedcontents:
remove(c)
remove(prev)
prev.name = obj.name + ' ' + str(i)
def readd(o):
self.allobjects[o.fullName()] = o
for c in o.orderedcontents:
readd(c)
readd(prev)
self.allobjects[obj.fullName()] = obj
return obj
def getProcessedModule(self, modname):
mod = self.allobjects.get(modname)
if mod is None:
return None
if isinstance(mod, Package):
return self.getProcessedModule(modname + '.__init__').parent
if not isinstance(mod, Module):
return None
if mod.state == UNPROCESSED:
self.processModule(mod)
return mod
def processModule(self, mod):
assert mod.state == UNPROCESSED
mod.state = PROCESSING
if getattr(mod, 'filepath', None) is None:
return
builder = self.defaultBuilder(self)
ast = builder.parseFile(mod.filepath)
if ast:
self.processing_modules.append(mod.fullName())
self.msg("processModule", "processing %s"%(self.processing_modules), 1)
builder.processModuleAST(ast, mod)
mod.state = PROCESSED
head = self.processing_modules.pop()
assert head == mod.fullName()
self.unprocessed_modules.remove(mod)
self.progress(
'process',
self.module_count - len(self.unprocessed_modules),
self.module_count,
"modules processed %s warnings"%(
sum(len(v) for v in self.warnings.itervalues()),))
def process(self):
while self.unprocessed_modules:
mod = iter(self.unprocessed_modules).next()
self.processModule(mod)
def fetchIntersphinxInventories(self):
"""
Download and parse intersphinx inventories based on configuration.
"""
for url in self.options.intersphinx:
self.intersphinx.update(url)
