def compare_frozen(app, exception):
"""
Compare backed up frozen RST files from ``backup_frozen()`` with
newly frozen RST. If they don't match within accepted tolerances,
raise an exception to fail the current CI job. This implies that
the OCIO API changed without building with CMake option
``-DOCIO_BUILD_FROZEN_DOCS=ON``; needed to update frozen RST in the
source tree.
"""
# Raise Sphinx exceptions for debugging
if exception:
raise ExtensionError(str(exception))
if not app.config.frozendoc_compare:
return
logger.info("Comparing frozen RST: {src} <-> {dst}\n".format(
src=PYTHON_FROZEN_DIR, dst=PYTHON_BACKUP_DIR))
frozen_files = set(os.listdir(PYTHON_FROZEN_DIR))
backup_files = set(os.listdir(PYTHON_BACKUP_DIR))
# Find any files which are different, or only present in one of the two
# directories.
match, mismatch, errors = filecmp.cmpfiles(PYTHON_FROZEN_DIR,
PYTHON_BACKUP_DIR,
list(frozen_files
| backup_files),
shallow=False)
# Different OSs or compilers may result in slightly different signatures
# or types. Test each mismatched file for the ratio of how different
# their contents are. If they have a difference ratio at or above 0.6 (the
# recommended ratio to determine close matches), assume the differences are
# ok. This is certainly not a water tight assumption, but deemed (at the
# moment) preferrable to failing on every mior difference. Keep track of
# these ignored differences and report them for transparency in CI logs.
ignored = []
for i in reversed(range(len(mismatch))):
filename = mismatch[i]
logger.info("Difference found in: {}".format(filename))
frozen_path = os.path.join(PYTHON_FROZEN_DIR, filename)
with open(frozen_path, "r") as frozen_file:
frozen_data = frozen_file.read()
backup_path = os.path.join(PYTHON_BACKUP_DIR, filename)
with open(backup_path, "r") as backup_file:
backup_data = backup_file.read()
for line in difflib.unified_diff(frozen_data.splitlines(),
backup_data.splitlines(),
fromfile=frozen_path,
tofile=backup_path):
logger.info(line)
# Based on difflib's caution of argument order playing a role in the
# results of ratio(), check the ratio in both directions and use the
# better of the two as our heuristic.
match_ab = difflib.SequenceMatcher(None, frozen_data, backup_data)
match_ba = difflib.SequenceMatcher(None, backup_data, frozen_data)
max_ratio = max(match_ab.ratio(), match_ba.ratio())
if max_ratio >= 0.6:
ignored.append(mismatch.pop(i))
logger.info(
"Difference ratio {} is within error tolerances\n".format(
max_ratio))
else:
logger.error(
"Difference ratio {} exceeds error tolerances\n".format(
max_ratio))
if os.path.exists(PYTHON_BACKUP_DIR):
shutil.rmtree(PYTHON_BACKUP_DIR)
if mismatch or errors:
raise ExtensionError(
"Frozen RST is out of date! Build OpenColorIO with CMake option "
"'-DOCIO_BUILD_FROZEN_DOCS=ON' to update required frozen "
"documentation source files in: {dir}\n\n"
" Changed files: {changed}\n\n"
" Added files: {added}\n\n"
" Removed files: {removed}\n\n"
"See log for changed file differences.\n".format(
dir=PYTHON_FROZEN_DIR,
changed=", ".join(mismatch),
added=", ".join(f for f in errors if f in frozen_files),
removed=", ".join(f for f in errors if f in backup_files),
))
# Report ignored differences
if ignored:
logger.warning(
"Some differences were found and ignored when comparing frozen "
"RST to current documentation source in: {dir}\n\n"
" Changed files: {changed}\n\n"
"See log for changed file differences.\n".format(
dir=PYTHON_FROZEN_DIR,
changed=", ".join(ignored),
))
logger.info("Frozen RST matches within error tolerances.")
def add_role_to_domain(self, domain, name, role):
self.debug('[app] adding role to domain: %r', (domain, name, role))
if domain not in self.domains:
raise ExtensionError('domain %s not yet registered' % domain)
self.domains[domain].roles[name] = role
def _validate_event(self, event):
event = intern(event)
if event not in self._events:
raise ExtensionError('Unknown event name: %s' % event)
def add(self, name, default, rebuild, types):
# type: (unicode, Any, Union[bool, unicode], Any) -> None
if name in self.values:
raise ExtensionError(__('Config value %r already present') % name)
else:
self.values[name] = (default, rebuild, types)
def add(self, name: str) -> None:
"""Register a custom Sphinx event."""
if name in self.events:
raise ExtensionError(__('Event %r already present') % name)
self.events[name] = ''
def _validate_event(self, event):
# type: (unicode) -> None
if event not in self._events:
raise ExtensionError('Unknown event name: %s' % event)
def notation_to_string(notation):
"""Parse notation and format it as a string with ellipses."""
try:
return stringify_with_ellipses(notation)
except ParseError as e:
raise ExtensionError(PARSE_ERROR.format(notation, e.msg)) from e
mod.setup(self)
self._extensions[extension] = mod
def import_object(self, objname, source=None):
"""Import an object from a 'module.name' string."""
try:
module, name = objname.rsplit('.', 1)
except ValueError, err:
raise ExtensionError(
'Invalid full object name %s' % objname +
(source and ' (needed for %s)' % source or ''), err)
try:
return getattr(__import__(module, None, None, [name]), name)
except ImportError, err:
raise ExtensionError(
'Could not import %s' % module +
(source and ' (needed for %s)' % source or ''), err)
except AttributeError, err:
raise ExtensionError(
'Could not find %s' % objname +
(source and ' (needed for %s)' % source or ''), err)
# event interface
def _validate_event(self, event):
event = intern(event)
if event not in self._events:
raise ExtensionError('Unknown event name: %s' % event)
def connect(self, event, callback):
self._validate_event(event)
def add_config_value(self, name, default, rebuild):
if name in self.config.values:
raise ExtensionError('Config value %r already present' % name)
if rebuild in (False, True):
rebuild = rebuild and 'env' or ''
self.config.values[name] = (default, rebuild)
def save_thumbnail(image_path_template, src_file, script_vars, file_conf,
gallery_conf):
"""Generate and Save the thumbnail image
Parameters
----------
image_path_template : str
holds the template where to save and how to name the image
src_file : str
path to source python file
script_vars : dict
Configuration and run time variables
file_conf : dict
File-specific settings given in source file comments as:
``# sphinx_gallery_<name> = <value>``
gallery_conf : dict
Sphinx-Gallery configuration dictionary
"""
thumb_dir = os.path.join(os.path.dirname(image_path_template), 'thumb')
if not os.path.exists(thumb_dir):
os.makedirs(thumb_dir)
# read specification of the figure to display as thumbnail from main text
thumbnail_number = file_conf.get('thumbnail_number', None)
thumbnail_path = file_conf.get('thumbnail_path', None)
# thumbnail_number has priority.
if thumbnail_number is None and thumbnail_path is None:
# If no number AND no path, set to default thumbnail_number
thumbnail_number = 1
if thumbnail_number is None:
image_path = os.path.join(gallery_conf['src_dir'], thumbnail_path)
else:
if not isinstance(thumbnail_number, int):
raise ExtensionError(
'sphinx_gallery_thumbnail_number setting is not a number, '
'got %r' % (thumbnail_number,))
# negative index means counting from the last one
if thumbnail_number < 0:
thumbnail_number += len(script_vars["image_path_iterator"]) + 1
image_path = image_path_template.format(thumbnail_number)
del thumbnail_number, thumbnail_path, image_path_template
thumbnail_image_path, ext = _find_image_ext(image_path)
base_image_name = os.path.splitext(os.path.basename(src_file))[0]
thumb_file = os.path.join(thumb_dir,
'sphx_glr_%s_thumb.%s' % (base_image_name, ext))
if src_file in gallery_conf['failing_examples']:
img = os.path.join(glr_path_static(), 'broken_example.png')
elif os.path.exists(thumbnail_image_path):
img = thumbnail_image_path
elif not os.path.exists(thumb_file):
# create something to replace the thumbnail
img = gallery_conf.get("default_thumb_file")
if img is None:
img = os.path.join(glr_path_static(), 'no_image.png')
else:
return
if ext in ('svg', 'gif'):
copyfile(img, thumb_file)
else:
scale_image(img, thumb_file, *gallery_conf["thumbnail_size"])
if 'thumbnails' in gallery_conf['compress_images']:
optipng(thumb_file, gallery_conf['compress_images_args'])
def _check_input(prompt=None):
raise ExtensionError(
'Cannot use input() builtin function in Sphinx-Gallery examples')
def check_config(app):
if not app.config.googleanalytics_id:
raise ExtensionError(
"'googleanalytics_id' config value must be set for ga statistics to function properly."
)
def add_node(self, node, **kwds):
# type: (nodes.Node, Any) -> None
"""Register a Docutils node class.
This is necessary for Docutils internals. It may also be used in the
future to validate nodes in the parsed documents.
Node visitor functions for the Sphinx HTML, LaTeX, text and manpage
writers can be given as keyword arguments: the keyword should be one or
more of ``'html'``, ``'latex'``, ``'text'``, ``'man'``, ``'texinfo'``
or any other supported translators, the value a 2-tuple of ``(visit,
depart)`` methods. ``depart`` can be ``None`` if the ``visit``
function raises :exc:`docutils.nodes.SkipNode`. Example:
.. code-block:: python
class math(docutils.nodes.Element): pass
def visit_math_html(self, node):
self.body.append(self.starttag(node, 'math'))
def depart_math_html(self, node):
self.body.append('</math>')
app.add_node(math, html=(visit_math_html, depart_math_html))
Obviously, translators for which you don't specify visitor methods will
choke on the node when encountered in a document to translate.
.. versionchanged:: 0.5
Added the support for keyword arguments giving visit functions.
"""
logger.debug('[app] adding node: %r', (node, kwds))
if not kwds.pop('override', False) and \
hasattr(nodes.GenericNodeVisitor, 'visit_' + node.__name__):
logger.warning(__('while setting up extension %s: node class %r is '
'already registered, its visitors will be overridden'),
self._setting_up_extension, node.__name__,
type='app', subtype='add_node')
nodes._add_node_class_names([node.__name__])
for key, val in iteritems(kwds):
try:
visit, depart = val
except ValueError:
raise ExtensionError(__('Value for key %r must be a '
'(visit, depart) function tuple') % key)
translator = self.registry.translators.get(key)
translators = []
if translator is not None:
translators.append(translator)
elif key == 'html':
from sphinx.writers.html import HTMLTranslator
translators.append(HTMLTranslator)
if is_html5_writer_available():
from sphinx.writers.html5 import HTML5Translator
translators.append(HTML5Translator)
elif key == 'latex':
from sphinx.writers.latex import LaTeXTranslator
translators.append(LaTeXTranslator)
elif key == 'text':
from sphinx.writers.text import TextTranslator
translators.append(TextTranslator)
elif key == 'man':
from sphinx.writers.manpage import ManualPageTranslator
translators.append(ManualPageTranslator)
elif key == 'texinfo':
from sphinx.writers.texinfo import TexinfoTranslator
translators.append(TexinfoTranslator)
for translator in translators:
setattr(translator, 'visit_' + node.__name__, visit)
if depart:
setattr(translator, 'depart_' + node.__name__, depart)
def add_enumerable_node(self, node, figtype, title_getter=None, override=False):
# type: (nodes.Node, unicode, TitleGetter, bool) -> None
logger.debug('[app] adding enumerable node: (%r, %r, %r)', node, figtype, title_getter)
if node in self.enumerable_nodes and not override:
raise ExtensionError(__('enumerable_node %r already registered') % node)
self.enumerable_nodes[node] = (figtype, title_getter)
def add_source_parser(self, suffix, parser):
# type: (unicode, Type[Parser]) -> None
logger.debug('[app] adding search source_parser: %r, %r', suffix, parser)
if suffix in self.source_parsers:
raise ExtensionError(__('source_parser for %r is already registered') % suffix)
self.source_parsers[suffix] = parser
def add_event(self, name):
if name in self._events:
raise ExtensionError('Event %r already present' % name)
self._events[name] = ''
def pymol_scraper(block, block_vars, gallery_conf):
# Search for comment line containing 'Visualization with PyMOL...'
_, code, _ = block
if any([
line.strip() == "# Visualization with PyMOL..."
for line in code.splitlines()
]):
pymol_script_path = splitext(block_vars["src_file"])[0] + "_pymol.py"
# The rendered image will be created in the same directory as
# the example script
# -> the image will be included in version control
# -> Rendering with PyMOL is not necessary for building the docs
pymol_image_path = splitext(block_vars["src_file"])[0] + ".png"
if not isfile(pymol_script_path):
raise ExtensionError(
f"'{block_vars['src_file']}' has no corresponding "
f"'{pymol_script_path}' file")
# If PyMOL image is already created, do not run PyMOL script,
# as this should not be required for building the documentation
if not isfile(pymol_image_path):
# Create a shallow copy,
# to avoid ading new variables to example script
script_globals = copy.copy(block_vars["example_globals"])
script_globals["__image_destination__"] = pymol_image_path
try:
import pymol
except ImportError:
raise ExtensionError("PyMOL is not installed")
try:
import ammolite
except ImportError:
raise ExtensionError("Ammolite is not installed")
with open(pymol_script_path, "r") as script:
# Prevent PyMOL from writing stuff (splash screen, etc.)
# to STDOUT or STDERR
# -> Save original STDOUT/STDERR and point them
# temporarily to DEVNULL
dev_null = open(os.devnull, 'w')
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = dev_null
sys.stderr = dev_null
try:
exec(script.read(), script_globals)
except Exception as e:
raise ExtensionError(
f"PyMOL script raised a {type(e).__name__}: {str(e)}")
finally:
# Restore STDOUT/STDERR
sys.stdout = orig_stdout
sys.stderr = orig_stderr
dev_null.close()
if not isfile(pymol_image_path):
raise ExtensionError("PyMOL script did not create an image "
"(at expected location)")
# Copy the images into the 'gallery' directory under a canonical
# sphinx-gallery name
image_path_iterator = block_vars['image_path_iterator']
image_destination = image_path_iterator.next()
shutil.copy(pymol_image_path, image_destination)
return figure_rst([image_destination], gallery_conf['src_dir'])
else:
return figure_rst([], gallery_conf['src_dir'])
def add_domain(self, domain: "Type[Domain]", override: bool = False) -> None:
logger.debug('[app] adding domain: %r', domain)
if domain.name in self.domains and not override:
raise ExtensionError(__('domain %s already registered') % domain.name)
self.domains[domain.name] = domain
def add(self, name: str, default: Any, rebuild: Union[bool, str],
types: Any) -> None:
if name in self.values:
raise ExtensionError(__('Config value %r already present') % name)
else:
self.values[name] = (default, rebuild, types)
def add_source_suffix(self, suffix: str, filetype: str, override: bool = False) -> None:
logger.debug('[app] adding source_suffix: %r, %r', suffix, filetype)
if suffix in self.source_suffix and not override:
raise ExtensionError(__('source_suffix %r is already registered') % suffix)
else:
self.source_suffix[suffix] = filetype
def add_source_input(self, filetype, input_class):
# type: (unicode, Type[Input]) -> None
if filetype in self.source_inputs:
raise ExtensionError(
__('source_input for %r is already registered') % filetype)
self.source_inputs[filetype] = input_class
def add_translator(self, name: str, translator: "Type[nodes.NodeVisitor]",
override: bool = False) -> None:
logger.debug('[app] Change of translator for the %s builder.', name)
if name in self.translators and not override:
raise ExtensionError(__('Translator for %r already exists') % name)
self.translators[name] = translator
def run_autoapi(app): # pylint: disable=too-many-branches
"""
Load AutoAPI data from the filesystem.
"""
if app.config.autoapi_type not in LANGUAGE_MAPPERS:
raise ExtensionError(
"Invalid autoapi_type setting, "
"following values is allowed: {}".format(", ".join(
'"{}"'.format(api_type)
for api_type in sorted(LANGUAGE_MAPPERS))))
if not app.config.autoapi_dirs:
raise ExtensionError("You must configure an autoapi_dirs setting")
if app.config.autoapi_include_summaries is not None:
warnings.warn(
"autoapi_include_summaries has been replaced by "
"the show-module-summary AutoAPI option\n",
RemovedInAutoAPI2Warning,
)
if app.config.autoapi_include_summaries:
app.config.autoapi_options.append("show-module-summary")
# Make sure the paths are full
normalised_dirs = _normalise_autoapi_dirs(app.config.autoapi_dirs,
app.srcdir)
for _dir in normalised_dirs:
if not os.path.exists(_dir):
raise ExtensionError(
"AutoAPI Directory `{dir}` not found. "
"Please check your `autoapi_dirs` setting.".format(dir=_dir))
normalized_root = os.path.normpath(
os.path.join(app.srcdir, app.config.autoapi_root))
url_root = os.path.join("/", app.config.autoapi_root)
if not all(import_name in sys.modules for _, import_name in
LANGUAGE_REQUIREMENTS[app.config.autoapi_type]):
raise ExtensionError(
"AutoAPI of type `{type}` requires following "
"packages to be installed and included in extensions list: "
"{packages}".format(
type=app.config.autoapi_type,
packages=", ".join(
'{import_name} (available as "{pkg_name}" on PyPI)'.format(
pkg_name=pkg_name, import_name=import_name)
for pkg_name, import_name in LANGUAGE_REQUIREMENTS[
app.config.autoapi_type]),
))
sphinx_mapper = LANGUAGE_MAPPERS[app.config.autoapi_type]
template_dir = app.config.autoapi_template_dir
if template_dir and not os.path.isabs(template_dir):
if not os.path.isdir(template_dir):
template_dir = os.path.join(app.srcdir,
app.config.autoapi_template_dir)
elif app.srcdir != os.getcwd():
warnings.warn(
"autoapi_template_dir will be expected to be "
"relative to the Sphinx source directory instead of "
"relative to where sphinx-build is run\n",
RemovedInAutoAPI2Warning,
)
sphinx_mapper_obj = sphinx_mapper(app,
template_dir=template_dir,
url_root=url_root)
if app.config.autoapi_file_patterns:
file_patterns = app.config.autoapi_file_patterns
else:
file_patterns = DEFAULT_FILE_PATTERNS.get(app.config.autoapi_type, [])
if app.config.autoapi_ignore:
ignore_patterns = app.config.autoapi_ignore
else:
ignore_patterns = DEFAULT_IGNORE_PATTERNS.get(app.config.autoapi_type,
[])
if ".rst" in app.config.source_suffix:
out_suffix = ".rst"
elif ".txt" in app.config.source_suffix:
out_suffix = ".txt"
else:
# Fallback to first suffix listed
out_suffix = next(iter(app.config.source_suffix))
if sphinx_mapper_obj.load(patterns=file_patterns,
dirs=normalised_dirs,
ignore=ignore_patterns):
sphinx_mapper_obj.map(options=app.config.autoapi_options)
if app.config.autoapi_generate_api_docs:
sphinx_mapper_obj.output_rst(root=normalized_root,
source_suffix=out_suffix)
def add_enumerable_node(self, node: "Type[Node]", figtype: str,
title_getter: TitleGetter = None, override: bool = False) -> None:
logger.debug('[app] adding enumerable node: (%r, %r, %r)', node, figtype, title_getter)
if node in self.enumerable_nodes and not override:
raise ExtensionError(__('enumerable_node %r already registered') % node)
self.enumerable_nodes[node] = (figtype, title_getter)
def add_domain(self, domain):
self.debug('[app] adding domain: %r', domain)
if domain.name in self.domains:
raise ExtensionError('domain %s already registered' % domain.name)
self.domains[domain.name] = domain
def add_translator(self, name, translator, override=False):
# type: (unicode, Type[nodes.NodeVisitor], bool) -> None
logger.debug('[app] Change of translator for the %s builder.' % name)
if name in self.translators and not override:
raise ExtensionError(__('Translator for %r already exists') % name)
self.translators[name] = translator
def add_index_to_domain(self, domain, index):
self.debug('[app] adding index to domain: %r', (domain, index))
if domain not in self.domains:
raise ExtensionError('domain %s not yet registered' % domain)
self.domains[domain].indices.append(index)
def add_domain(self, domain):
# type: (Type[Domain]) -> None
logger.debug('[app] adding domain: %r', domain)
if domain.name in self.domains:
raise ExtensionError(__('domain %s already registered') % domain.name)
self.domains[domain.name] = domain
def add_event(self, name):
self.debug('[app] adding event: %r', name)
if name in self._events:
raise ExtensionError('Event %r already present' % name)
self._events[name] = ''
def add_index_to_domain(self, domain, index):
# type: (unicode, Type[Index]) -> None
logger.debug('[app] adding index to domain: %r', (domain, index))
if domain not in self.domains:
raise ExtensionError(_('domain %s not yet registered') % domain)
self.domains[domain].indices.append(index)
