def test_EpydocLinker_resolve_identifier_xref_intersphinx_relative_id(
) -> None:
"""
Return the link from inventory using short names, by resolving them based
on the imports done in the module.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['ext_package.ext_module'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
# Here we set up the target module as it would have this import.
# from ext_package import ext_module
ext_package = model.Module(system, 'ext_package')
target.contents['ext_module'] = model.Module(system,
'ext_module',
parent=ext_package)
sut = epydoc2stan._EpydocLinker(target)
# This is called for the L{ext_module<Pretty Text>} markup.
url = sut.resolve_identifier('ext_module')
url_xref = sut.resolve_identifier_xref('ext_module', 0)
assert "http://tm.tld/some.html" == url
assert "http://tm.tld/some.html" == url_xref
def test_generate_empty_functional():
"""
Functional test for index generation of empty API.
Header is plain text while content is compressed.
"""
project_name = 'some-name'
log = []
logger = lambda section, message, thresh=0: log.append((
section, message, thresh))
sut = SphinxInventory(logger=logger, project_name=project_name)
output = PersistentStringIO()
sut._openFileForWriting = lambda path: closing(output)
sut.generate(subjects=[], basepath='base-path')
expected_log = [(
'sphinx',
'Generating objects inventory at base-path/objects.inv',
0
)]
assert expected_log == log
expected_ouput = """# Sphinx inventory version 2
# Project: some-name
# Version: 2.0
# The rest of this file is compressed with zlib.
x\x9c\x03\x00\x00\x00\x00\x01"""
assert expected_ouput == output.getvalue()
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options: Optional[Values] = None):
self.allobjects: Dict[str, Documentable] = {}
self.rootobjects: List[Documentable] = []
self.violations = 0
"""The number of docstring problems found.
This is used to determine whether to fail the build when using
the --warnings-as-errors option, so it should only be increased
for problems that the user can fix.
"""
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.projectname = 'my project'
self.docstring_syntax_errors: Set[str] = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs: Set[Tuple[str, str]] = set()
self.unprocessed_modules: Set[Module] = set()
self.module_count = 0
self.processing_modules: List[str] = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def test_EpydocLinker_translate_identifier_xref_intersphinx_relative_id():
"""
Return the link from inventory using short names, by resolving them based
on the imports done in the module.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["ext_package.ext_module"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
# Here we set up the target module as it would have this import.
# from ext_package import ext_module
ext_package = model.Module(system, "ext_package", "ignore-docstring")
target.contents["ext_module"] = model.Module(system,
"ext_module",
"ignore-docstring",
parent=ext_package)
sut = epydoc2stan._EpydocLinker(target)
# This is called for the L{ext_module<Pretty Text>} markup.
result = sut.translate_identifier_xref("ext_module", "Pretty Text")
expected = '<a href="http://tm.tld/some.html"><code>Pretty Text</code></a>'
assert expected == result
def __init__(self, options=None):
self.allobjects = OrderedDict()
self.rootobjects = []
self.warnings = {}
self.packages = []
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.docstring_syntax_errors = set()
"""FullNames of objects for which the docstring failed to parse."""
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options=None):
self.allobjects = {}
self.orderedallobjects = []
self.rootobjects = []
self.warnings = {}
self.packages = []
self.moresystems = []
self.subsystems = []
self.urlprefix = ''
if options:
self.options = options
else:
from pydoctor.driver import parse_args
self.options, _ = parse_args([])
self.options.verbosity = 3
self.abbrevmapping = {}
self.projectname = 'my project'
self.epytextproblems = [
] # fullNames of objects that failed to epytext properly
self.verboselevel = 0
self.needsnl = False
self.once_msgs = set()
self.unprocessed_modules = set()
self.module_count = 0
self.processing_modules = []
self.buildtime = datetime.datetime.now()
# Once pickle support is removed, System should be
# initialized with project name so that we can reuse intersphinx instance for
# object.inv generation.
self.intersphinx = SphinxInventory(logger=self.msg,
project_name=self.projectname)
def test_initialization():
"""
Is initialized with logger and project name.
"""
logger = object()
name = object()
sut = SphinxInventory(logger=logger, project_name=name)
assert logger is sut.info
assert name is sut.project_name
def test_EpydocLinker_look_for_intersphinx_hit() -> None:
"""
Return the link from inventory based on first package name.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
sut = epydoc2stan._EpydocLinker(target)
result = sut.look_for_intersphinx('base.module.other')
assert 'http://tm.tld/some.html' == result
def test_EpydocLinker_look_for_intersphinx_hit():
"""
Return the link from inventory based on first package name.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["base.module.other"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
sut = epydoc2stan._EpydocLinker(target)
result = sut.look_for_intersphinx("base.module.other")
assert "http://tm.tld/some.html" == result
def test_xref_link_intersphinx() -> None:
"""A linked name that is documented in another project is linked using
an absolute URL (retrieved via Intersphinx).
"""
mod = fromText('''
def func():
"""This is a thin wrapper around L{external.func}."""
''', modname='test')
system = mod.system
inventory = SphinxInventory(system.msg)
inventory._links['external.func'] = ('https://example.net', 'lib.html#func')
system.intersphinx = inventory
html = docstring2html(mod.contents['func'])
assert 'href="https://example.net/lib.html#func"' in html
def test_EpydocLinker_resolve_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name')
sut = epydoc2stan._EpydocLinker(target)
url = sut.resolve_identifier_xref('base.module.other')
assert "http://tm.tld/some.html" == url
def test_EpydocLinker_translate_identifier_xref_intersphinx():
"""
Return the link from inventory.
"""
system = model.System()
inventory = SphinxInventory(system.msg, 'some-project')
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name', 'ignore-docstring')
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref(
'base.module.other', 'base.module.pretty')
expected = (
'<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
)
assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg, "some-project")
inventory._links["base.module.other"] = ("http://tm.tld", "some.html")
system.intersphinx = inventory
target = model.Module(system, "ignore-name", "ignore-docstring")
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref("base.module.other",
"base.module.pretty")
expected = '<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id():
"""
Returns the link from Sphinx inventory based on a cross reference
ID specified in absolute dotted path and with a custom pretty text for the
URL.
"""
system = model.System()
inventory = SphinxInventory(system.msg)
inventory._links['base.module.other'] = ('http://tm.tld', 'some.html')
system.intersphinx = inventory
target = model.Module(system, 'ignore-name', 'ignore-docstring')
sut = epydoc2stan._EpydocLinker(target)
result = sut.translate_identifier_xref('base.module.other',
'base.module.pretty')
expected = (
'<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>'
)
assert expected == flatten(result)
def __setstate__(self, state):
if "abbrevmapping" not in state:
state["abbrevmapping"] = {}
# this is so very, very evil.
# see doc/extreme-pickling-pain.txt for more.
def lookup(name):
for system in [self] + self.moresystems + self.subsystems:
if name in system.allobjects:
return system.allobjects[name]
raise KeyError(name)
self.__dict__.update(state)
for system in [self] + self.moresystems + self.subsystems:
if "allobjects" not in system.__dict__:
return
for system in [self] + self.moresystems + self.subsystems:
for obj in system.orderedallobjects:
for k, v in obj.__dict__.copy().iteritems():
if k.startswith("$"):
del obj.__dict__[k]
obj.__dict__[k[1:]] = lookup(v)
elif k.startswith("@"):
n = []
for vv in v:
if vv is None:
n.append(None)
else:
n.append(lookup(vv))
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
elif k.startswith("!"):
n = {}
for kk, vv in v.iteritems():
n[kk] = lookup(vv)
del obj.__dict__[k]
obj.__dict__[k[1:]] = n
self.intersphinx = SphinxInventory(
logger=self.msg, project_name=self.projectname
)
def main(args):
import cPickle
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
if options.inputpickle:
system = cPickle.load(open(options.inputpickle, 'rb'))
if options.systemclass:
if type(system) is not systemclass:
cls = type(system)
msg = ("loaded pickle has class %s.%s, differing "
"from explicitly requested %s")
error(msg, cls.__module__, cls.__name__,
options.systemclass)
else:
system = systemclass()
# Once pickle support is removed, always instantiate System with
# options and make fetchIntersphinxInventories private in __init__.
system.options = options
system.fetchIntersphinxInventories()
system.urlprefix = ''
if options.moresystems:
moresystems = []
for fnamepref in options.moresystems:
fname, prefix = fnamepref.split(':', 1)
moresystems.append(cPickle.load(open(fname, 'rb')))
moresystems[-1].urlprefix = prefix
moresystems[-1].options = system.options
moresystems[-1].subsystems.append(system)
system.moresystems = moresystems
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
if options.auto:
options.server = True
options.edit = True
for fn in os.listdir('.'):
if os.path.isdir(fn) and \
os.path.exists(os.path.join(fn, '__init__.py')):
options.packages.append(fn)
elif fn.endswith('.py') and fn != 'setup.py':
options.modules.append(fn)
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.outputpickle and not options.testing \
and not options.server and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError, e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(
system, m, None, prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', None, prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.livecheck:
error("write this")
if system.options.projectname is None:
name = '/'.join([ro.name for ro in system.rootobjects])
system.msg(
'warning',
'WARNING: guessing '+name+' for project name', thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: save the system, if desired
if options.outputpickle:
system.msg('', 'saving output pickle to ' + options.outputpickle)
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
# step 5: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(
options.htmlwriter, '--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg('html', 'writing html to %s using %s.%s'%(
options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
system.epytextproblems = []
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.epytextproblems:
def p(msg):
system.msg(('epytext', 'epytext-summary'), msg, thresh=-1, topthresh=1)
p("these %s objects' docstrings are not proper epytext:"
%(len(system.epytextproblems),))
exitcode = 2
for fn in system.epytextproblems:
p(' '+fn)
if options.outputpickle:
system.msg(
'', 'saving output pickle to ' + options.outputpickle)
# save again, with epytextproblems
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventory(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
# Finally, if we should serve html, lets serve some html.
if options.server:
from pydoctor.server import (
EditingPyDoctorResource, PyDoctorResource)
from pydoctor.epydoc2stan import doc2stan
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.web.vhost import VHostMonsterResource
from twisted.internet import reactor
if options.edit:
if not options.nocheck:
system.msg(
"server", "Checking formatting of docstrings.")
included_obs = [
ob for ob in system.orderedallobjects
if ob.isVisible]
for i, ob in enumerate(included_obs):
system.progress(
"server", i+1, len(included_obs),
"docstrings checked, found %s problems" % (
len(system.epytextproblems)))
doc2stan(ob, docstring=ob.docstring)
root = EditingPyDoctorResource(system)
else:
root = PyDoctorResource(system)
if options.facing_path:
options.local_only = True
realroot = Resource()
cur = realroot
realroot.putChild('vhost', VHostMonsterResource())
segments = options.facing_path.split('/')
for segment in segments[:-1]:
next = Resource()
cur.putChild(segment, next)
cur = next
cur.putChild(segments[-1], root)
root = realroot
system.msg(
"server",
"Setting up server at http://localhost:%d/" %
options.server_port)
if options.auto:
def wb_open():
import webbrowser
webbrowser.open(
'http://localhost:%d/' % options.server_port)
reactor.callWhenRunning(wb_open)
from twisted.python import log
log.startLogging(sys.stdout)
site = Site(root)
if options.local_only:
interface = 'localhost'
else:
interface = ''
reactor.listenTCP(options.server_port, site, interface=interface)
reactor.run()
def main(args):
import cPickle
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
if options.inputpickle:
system = cPickle.load(open(options.inputpickle, 'rb'))
if options.systemclass:
if type(system) is not systemclass:
cls = type(system)
msg = ("loaded pickle has class %s.%s, differing "
"from explicitly requested %s")
error(msg, cls.__module__, cls.__name__,
options.systemclass)
else:
system = systemclass()
# Once pickle support is removed, always instantiate System with
# options and make fetchIntersphinxInventories private in __init__.
system.options = options
system.fetchIntersphinxInventories(cache)
system.urlprefix = ''
if options.moresystems:
moresystems = []
for fnamepref in options.moresystems:
fname, prefix = fnamepref.split(':', 1)
moresystems.append(cPickle.load(open(fname, 'rb')))
moresystems[-1].urlprefix = prefix
moresystems[-1].options = system.options
moresystems[-1].subsystems.append(system)
system.moresystems = moresystems
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.outputpickle and not options.testing \
and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(e)
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(system, m, None,
prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', None,
prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.livecheck:
error("write this")
if system.options.projectname is None:
name = '/'.join([ro.name for ro in system.rootobjects])
system.msg('warning',
'WARNING: guessing ' + name + ' for project name',
thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: save the system, if desired
if options.outputpickle:
system.msg('', 'saving output pickle to ' + options.outputpickle)
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
# step 5: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(options.htmlwriter,
'--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg(
'html', 'writing html to %s using %s.%s' %
(options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
system.epytextproblems = []
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.epytextproblems:
def p(msg):
system.msg(('epytext', 'epytext-summary'),
msg,
thresh=-1,
topthresh=1)
p("these %s objects' docstrings are not proper epytext:" %
(len(system.epytextproblems), ))
exitcode = 2
for fn in system.epytextproblems:
p(' ' + fn)
if options.outputpickle:
system.msg('',
'saving output pickle to ' + options.outputpickle)
# save again, with epytextproblems
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventory(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_traceback)
raise
return exitcode
def make_SphinxInventory(logger=object()):
"""
Return a SphinxInventory.
"""
return SphinxInventory(logger=logger, project_name='project_name')
