def test_prepareCache(tmpdir, clearCache, enableCache, cacheDirectoryName,
maxAgeAmount, maxAgeUnit):
"""
The cache directory is deleted when C{clearCache} is L{True}; an
L{IntersphinxCache} is created with a session on which is mounted
L{cachecontrol.CacheControlAdapter} for C{http} and C{https} URLs.
"""
cacheDirectory = tmpdir.join("fakecache").ensure(dir=True)
for child in cacheDirectory.listdir():
child.remove()
cacheDirectory.ensure(cacheDirectoryName)
try:
cache = sphinx.prepareCache(
clearCache=clearCache,
enableCache=enableCache,
cachePath=str(cacheDirectory),
maxAge="{}{}".format(maxAgeAmount, maxAgeUnit),
)
except sphinx.InvalidMaxAge:
pass
else:
assert isinstance(cache, sphinx.IntersphinxCache)
for scheme in ("https://", "http://"):
hasCacheControl = isinstance(cache._session.adapters[scheme],
cachecontrol.CacheControlAdapter)
if enableCache:
assert hasCacheControl
else:
assert not hasCacheControl
if clearCache:
assert not tmpdir.listdir()
def test_prepareCache(
cacheDirectory: Path,
clearCache: bool,
enableCache: bool,
cacheDirectoryName: str,
maxAgeAmount: int,
maxAgeUnit: str,
) -> None:
"""
The cache directory is deleted when C{clearCache} is L{True}; an
L{IntersphinxCache} is created with a session on which is mounted
C{cachecontrol.CacheControlAdapter} for C{http} and C{https} URLs.
"""
# Windows doesn't like paths ending in a space or dot.
assume(cacheDirectoryName[-1] not in '. ')
# These DOS device names still have special meaning in modern Windows.
assume(cacheDirectoryName.upper() not in {'CON', 'PRN', 'AUX', 'NUL'})
assume(not cacheDirectoryName.upper().startswith('COM'))
assume(not cacheDirectoryName.upper().startswith('LPT'))
cacheDirectory.mkdir(exist_ok=True)
for child in cacheDirectory.iterdir():
child.unlink()
with open(cacheDirectory / cacheDirectoryName, 'w'):
pass
try:
cache = sphinx.prepareCache(clearCache=clearCache,
enableCache=enableCache,
cachePath=str(cacheDirectory),
maxAge=f"{maxAgeAmount}{maxAgeUnit}")
except sphinx.InvalidMaxAge:
pass
else:
assert isinstance(cache, sphinx.IntersphinxCache)
for scheme in ('https://', 'http://'):
hasCacheControl = isinstance(
cache._session.adapters[scheme],
cachecontrol.CacheControlAdapter,
)
if enableCache:
assert hasCacheControl
else:
assert not hasCacheControl
if clearCache:
assert not cacheDirectory.exists()
def main(args=sys.argv[1:]):
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
system = systemclass(options)
system.fetchIntersphinxInventories(cache)
if options.htmlsourcebase:
if options.projectbasedirectory is None:
error("you must specify --project-base-dir "
"when using --html-viewsource-base")
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.testing and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(e)
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(
system, m, prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.projectname is None:
name = '/'.join(ro.name for ro in system.rootobjects)
system.msg(
'warning',
'WARNING: guessing '+name+' for project name', thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(
options.htmlwriter, '--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg('html', 'writing html to %s using %s.%s'%(
options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.docstring_syntax_errors:
def p(msg):
system.msg(('epytext', 'epytext-summary'), msg, thresh=-1, topthresh=1)
p("these %s objects' docstrings contain syntax errors:"
%(len(system.docstring_syntax_errors),))
exitcode = 2
for fn in sorted(system.docstring_syntax_errors):
p(' '+fn)
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventoryWriter(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_info()[2])
raise
return exitcode
def main(args):
import cPickle
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class')
if not issubclass(systemclass, model.System):
msg = "%s is not a subclass of model.System"
error(msg, systemclass)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
if options.inputpickle:
system = cPickle.load(open(options.inputpickle, 'rb'))
if options.systemclass:
if type(system) is not systemclass:
cls = type(system)
msg = ("loaded pickle has class %s.%s, differing "
"from explicitly requested %s")
error(msg, cls.__module__, cls.__name__,
options.systemclass)
else:
system = systemclass()
# Once pickle support is removed, always instantiate System with
# options and make fetchIntersphinxInventories private in __init__.
system.options = options
system.fetchIntersphinxInventories(cache)
system.urlprefix = ''
if options.moresystems:
moresystems = []
for fnamepref in options.moresystems:
fname, prefix = fnamepref.split(':', 1)
moresystems.append(cPickle.load(open(fname, 'rb')))
moresystems[-1].urlprefix = prefix
moresystems[-1].options = system.options
moresystems[-1].subsystems.append(system)
system.moresystems = moresystems
system.sourcebase = options.htmlsourcebase
if options.abbrevmapping:
for thing in options.abbrevmapping.split(','):
k, v = thing.split('=')
system.abbrevmapping[k] = v
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.outputpickle and not options.testing \
and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(e)
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(e)
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(system, m, None,
prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__', None,
prependedpackage)
system.addObject(initmodule)
for path in args:
path = os.path.abspath(path)
if path in system.packages:
continue
if os.path.isdir(path):
system.msg('addPackage', 'adding directory ' + path)
system.addPackage(path, prependedpackage)
else:
system.msg('addModuleFromPath', 'adding module ' + path)
system.addModuleFromPath(prependedpackage, path)
system.packages.append(path)
# step 3: move the system to the desired state
if not system.packages:
error("The system does not contain any code, did you "
"forget an --add-package?")
system.process()
if system.options.livecheck:
error("write this")
if system.options.projectname is None:
name = '/'.join([ro.name for ro in system.rootobjects])
system.msg('warning',
'WARNING: guessing ' + name + ' for project name',
thresh=-1)
system.projectname = name
else:
system.projectname = system.options.projectname
# step 4: save the system, if desired
if options.outputpickle:
system.msg('', 'saving output pickle to ' + options.outputpickle)
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
# step 5: make html, if desired
if options.makehtml:
options.makeintersphinx = True
if options.htmlwriter:
writerclass = findClassFromDottedName(options.htmlwriter,
'--html-writer')
else:
from pydoctor import templatewriter
writerclass = templatewriter.TemplateWriter
system.msg(
'html', 'writing html to %s using %s.%s' %
(options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.system = system
writer.prepOutputDirectory()
system.epytextproblems = []
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects, options.htmlfunctionpages)
if system.epytextproblems:
def p(msg):
system.msg(('epytext', 'epytext-summary'),
msg,
thresh=-1,
topthresh=1)
p("these %s objects' docstrings are not proper epytext:" %
(len(system.epytextproblems), ))
exitcode = 2
for fn in system.epytextproblems:
p(' ' + fn)
if options.outputpickle:
system.msg('',
'saving output pickle to ' + options.outputpickle)
# save again, with epytextproblems
del system.options # don't persist the options
f = open(options.outputpickle, 'wb')
cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL)
f.close()
system.options = options
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventory(
logger=system.msg,
project_name=system.projectname,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_traceback)
raise
return exitcode
def main(args: Sequence[str] = sys.argv[1:]) -> int:
"""
This is the console_scripts entry point for pydoctor CLI.
@param args: Command line arguments to run the CLI.
"""
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class',
model.System)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
system = systemclass(options)
system.fetchIntersphinxInventories(cache)
if options.htmlsourcebase:
if options.projectbasedirectory is None:
error("you must specify --project-base-dir "
"when using --html-viewsource-base")
system.sourcebase = options.htmlsourcebase
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.testing and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(str(e))
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(str(e))
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(system, m,
prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__',
prependedpackage)
system.addObject(initmodule)
added_paths = set()
for arg in args:
path = resolve_path(arg)
if path in added_paths:
continue
if options.projectbasedirectory is not None:
# Note: Path.is_relative_to() was only added in Python 3.9,
# so we have to use this workaround for now.
try:
path.relative_to(options.projectbasedirectory)
except ValueError as ex:
error(f"Source path lies outside base directory: {ex}")
if path.is_dir():
system.msg('addPackage', f"adding directory {path}")
if not (path / '__init__.py').is_file():
error(f"Source directory lacks __init__.py: {path}")
system.addPackage(path, prependedpackage)
elif path.is_file():
system.msg('addModuleFromPath', f"adding module {path}")
system.addModuleFromPath(path, prependedpackage)
elif path.exists():
error(f"Source path is neither file nor directory: {path}")
else:
error(f"Source path does not exist: {path}")
added_paths.add(path)
else:
error("No source paths given.")
# step 3: move the system to the desired state
if system.options.projectname is None:
name = '/'.join(system.root_names)
system.msg('warning',
f"Guessing '{name}' for project name.",
thresh=0)
system.projectname = name
else:
system.projectname = system.options.projectname
system.process()
# step 4: make html, if desired
if options.makehtml:
options.makeintersphinx = True
from pydoctor import templatewriter
if options.htmlwriter:
writerclass = findClassFromDottedName(options.htmlwriter,
'--html-writer', IWriter)
else:
writerclass = templatewriter.TemplateWriter
system.msg(
'html', 'writing html to %s using %s.%s' %
(options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer: IWriter
# Handle custom HTML templates
if system.options.templatedir:
custom_lookup = TemplateLookup()
try:
custom_lookup.add_templatedir(
Path(system.options.templatedir))
except UnsupportedTemplateVersion as e:
error(str(e))
try:
# mypy error: Cannot instantiate abstract class 'IWriter'
writer = writerclass(
options.htmloutput, # type: ignore[abstract]
template_lookup=custom_lookup)
except TypeError:
# Custom class does not accept 'template_lookup' argument.
writer = writerclass(
options.htmloutput) # type: ignore[abstract]
warnings.warn(
f"Writer '{writerclass.__name__}' does not support "
"HTML template customization with --template-dir.")
else:
writer = writerclass(
options.htmloutput) # type: ignore[abstract]
writer.prepOutputDirectory()
subjects: Sequence[model.Documentable] = ()
if options.htmlsubjects:
subjects = [
system.allobjects[fn] for fn in options.htmlsubjects
]
else:
writer.writeSummaryPages(system)
if not options.htmlsummarypages:
subjects = system.rootobjects
writer.writeIndividualFiles(subjects)
if system.docstring_syntax_errors:
def p(msg: str) -> None:
system.msg('docstring-summary',
msg,
thresh=-1,
topthresh=1)
p("these %s objects' docstrings contain syntax errors:" %
(len(system.docstring_syntax_errors), ))
exitcode = 2
for fn in sorted(system.docstring_syntax_errors):
p(' ' + fn)
if system.violations and options.warnings_as_errors:
# Update exit code if the run has produced warnings.
exitcode = 3
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventoryWriter(
logger=system.msg,
project_name=system.projectname,
project_version=system.options.projectversion,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_info()[2])
raise
return exitcode
