def test_status_iterator(app, status, warning):
logging.setup(app, status, warning)
# test for old_status_iterator
status.truncate(0)
yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... '))
output = strip_escseq(status.getvalue())
assert 'testing ... hello sphinx world \n' in output
assert yields == ['hello', 'sphinx', 'world']
# test for status_iterator (verbosity=0)
status.truncate(0)
yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... ',
length=3, verbosity=0))
output = strip_escseq(status.getvalue())
assert 'testing ... [ 33%] hello \r' in output
assert 'testing ... [ 66%] sphinx \r' in output
assert 'testing ... [100%] world \r\n' in output
assert yields == ['hello', 'sphinx', 'world']
# test for status_iterator (verbosity=1)
status.truncate(0)
yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... ',
length=3, verbosity=1))
output = strip_escseq(status.getvalue())
assert 'testing ... [ 33%] hello\n' in output
assert 'testing ... [ 66%] sphinx\n' in output
assert 'testing ... [100%] world\n\n' in output
assert yields == ['hello', 'sphinx', 'world']
def _write_parallel(self, docnames, nproc):
# type: (Sequence[unicode], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> None
self.app.phase = BuildPhase.WRITING
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
self.app.phase = BuildPhase.RESOLVING
for chunk in status_iterator(chunks, __('writing output... '), "darkgreen",
len(chunks), self.app.verbosity):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg)
# make sure all threads have finished
logger.info(bold(__('waiting for workers...')))
tasks.join()
def install_collapse_static_files(app, env):
STATICS_DIR_PATH = os.path.join(app.builder.outdir, IMAGE_DIR_NAME)
dest_path = os.path.join(STATICS_DIR_PATH, 'sphinx-needs')
source_folder = os.path.join(os.path.dirname(__file__), "libs/html")
files_to_copy = [os.path.join(source_folder, "sphinx_needs_collapse.js")]
if parse_version(sphinx_version) < parse_version("1.6"):
global status_iterator
status_iterator = app.status_iterator
for source_file_path in status_iterator(
files_to_copy,
'Copying static files for sphinx-needs collapse support...', brown,
len(files_to_copy)):
if not os.path.isabs(source_file_path):
raise IOError(
"Path must be absolute. Got: {}".format(source_file_path))
if not os.path.exists(source_file_path):
raise IOError("File not found: {}".format(source_file_path))
dest_file_path = os.path.join(
dest_path, os.path.relpath(source_file_path, source_folder))
if not os.path.exists(os.path.dirname(dest_file_path)):
ensuredir(os.path.dirname(dest_file_path))
copyfile(source_file_path, dest_file_path)
safe_remove_file("sphinx-needs/libs/html/sphinx_needs_collapse.js",
app)
safe_add_file("sphinx-needs/libs/html/sphinx_needs_collapse.js", app)
def _read_parallel(self, docnames, app, nproc):
# type: (List[unicode], Sphinx, int) -> None
# clear all outdated docs at once
for docname in docnames:
app.emit('env-purge-doc', self, docname)
self.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> unicode
self.app = app
with sphinx_smartquotes_action(self):
for docname in docs:
self.read_doc(docname, app)
# allow pickling self to send it back
return BuildEnvironment.dumps(self)
def merge(docs, otherenv):
# type: (List[unicode], unicode) -> None
env = BuildEnvironment.loads(otherenv)
self.merge_info_from(docs, env, app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def _write_parallel(self, docnames, nproc):
# type: (Sequence[unicode], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> None
self.app.phase = BuildPhase.WRITING
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
self.app.phase = BuildPhase.RESOLVING
for chunk in status_iterator(chunks,
__('writing output... '), "darkgreen",
len(chunks), self.app.verbosity):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg)
# make sure all threads have finished
logger.info(bold(__('waiting for workers...')))
tasks.join()
def _read_parallel(self, docnames, nproc):
# type: (List[unicode], int) -> None
# clear all outdated docs at once
for docname in docnames:
self.app.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> bytes
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs, otherenv):
# type: (List[unicode], bytes) -> None
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def copy_resources(self):
"""Copy supporting resources to the output folder."""
resource_iterator = status_iterator(
self.resources.items(),
"copying resources... ",
"brown",
len(self.resources),
self.app.verbosity,
stringify_func=lambda r: r[0],
)
for dest, (op, value) in resource_iterator:
logger.debug("[tutorial]: %s: (%s, %s)", dest, op, value)
destination = pathlib.Path(self.outdir, dest)
if not destination.parent.exists():
destination.parent.mkdir(parents=True)
if op == "copy":
copyfile(str(value), str(destination))
continue
if op == "create":
with destination.open("w") as f:
f.write(value)
continue
raise TypeError(f"Unknown resource operation: '{op}'")
def install_backend_static_files(app, env):
STATICS_DIR_PATH = os.path.join(app.builder.outdir, STATICS_DIR_NAME)
dest_path = os.path.join(
STATICS_DIR_PATH, 'sphinxcontrib-images',
app.sphinxcontrib_images_backend.__class__.__name__)
files_to_copy = app.sphinxcontrib_images_backend.STATIC_FILES
for source_file_path in status_iterator(
files_to_copy, 'Copying static files for sphinxcontrib-images...',
brown, len(files_to_copy)):
dest_file_path = os.path.join(dest_path, source_file_path)
if not os.path.exists(os.path.dirname(dest_file_path)):
ensuredir(os.path.dirname(dest_file_path))
source_file_path = os.path.join(
os.path.dirname(sys.modules[app.sphinxcontrib_images_backend.
__class__.__module__].__file__),
source_file_path)
copyfile(source_file_path, dest_file_path)
if dest_file_path.endswith('.js'):
app.add_javascript(
os.path.relpath(dest_file_path, STATICS_DIR_PATH))
elif dest_file_path.endswith('.css'):
app.add_stylesheet(
os.path.relpath(dest_file_path, STATICS_DIR_PATH))
def copy_image_files(self) -> None:
if self.images:
stringify_func = ImageAdapter(self.app.env).get_original_image_uri
for src in status_iterator(self.images,
__('copying images... '),
"brown",
len(self.images),
self.app.verbosity,
stringify_func=stringify_func):
dest = self.images[src]
try:
copy_asset_file(path.join(self.srcdir, src),
path.join(self.outdir, dest))
except Exception as err:
logger.warning(__('cannot copy image file %r: %s'),
path.join(self.srcdir, src), err)
if self.config.latex_logo:
if not path.isfile(path.join(self.confdir,
self.config.latex_logo)):
raise SphinxError(
__('logo file %r does not exist') % self.config.latex_logo)
else:
copy_asset_file(
path.join(self.confdir, self.config.latex_logo),
self.outdir)
def _read_parallel(self, docnames, nproc):
# type: (List[unicode], int) -> None
# clear all outdated docs at once
for docname in docnames:
self.app.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> bytes
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs, otherenv):
# type: (List[unicode], bytes) -> None
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def install_styles_static_files(app, env):
STATICS_DIR_PATH = os.path.join(app.builder.outdir, IMAGE_DIR_NAME)
dest_path = os.path.join(STATICS_DIR_PATH, 'sphinx-needs')
files_to_copy = ["common.css"]
if app.config.needs_css == 'modern.css':
source_folder = os.path.join(os.path.dirname(__file__), "css/modern/")
for root, dirs, files in os.walk(source_folder):
for single_file in files:
files_to_copy.append(os.path.join(root, single_file))
elif app.config.needs_css == 'dark.css':
source_folder = os.path.join(os.path.dirname(__file__), "css/dark/")
for root, dirs, files in os.walk(source_folder):
for single_file in files:
files_to_copy.append(os.path.join(root, single_file))
elif app.config.needs_css == 'blank.css':
source_folder = os.path.join(os.path.dirname(__file__), "css/blank/")
for root, dirs, files in os.walk(source_folder):
for single_file in files:
files_to_copy.append(os.path.join(root, single_file))
else:
files_to_copy += [app.config.needs_css]
# Be sure no "old" css layout is already set
safe_remove_file("sphinx-needs/common.css", app)
safe_remove_file("sphinx-needs/blank.css", app)
safe_remove_file("sphinx-needs/modern.css", app)
safe_remove_file("sphinx-needs/dark.css", app)
if parse_version(sphinx_version) < parse_version("1.6"):
global status_iterator
status_iterator = app.status_iterator
for source_file_path in status_iterator(
files_to_copy,
'Copying static files for sphinx-needs custom style support...',
brown, len(files_to_copy)):
if not os.path.isabs(source_file_path):
source_file_path = os.path.join(os.path.dirname(__file__), "css",
source_file_path)
if not os.path.exists(source_file_path):
source_file_path = os.path.join(os.path.dirname(__file__), "css",
"blank", "blank.css")
logger.warning(
"{0} not found. Copying sphinx-internal blank.css".format(
source_file_path))
dest_file_path = os.path.join(dest_path,
os.path.basename(source_file_path))
if not os.path.exists(os.path.dirname(dest_file_path)):
ensuredir(os.path.dirname(dest_file_path))
copyfile(source_file_path, dest_file_path)
safe_add_file(os.path.relpath(dest_file_path, STATICS_DIR_PATH), app)
def _read_serial(self, docnames: List[str]) -> None:
for docname in status_iterator(docnames,
__('reading sources... '), "purple",
len(docnames), self.app.verbosity):
# remove all inventory entries for that file
self.events.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
self.read_doc(docname)
def _read_serial(self, docnames):
# type: (List[unicode]) -> None
for docname in status_iterator(docnames, 'reading sources... ', "purple",
len(docnames), self.app.verbosity):
# remove all inventory entries for that file
self.app.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
self.read_doc(docname)
def _write_serial(self, docnames):
# type: (Sequence[unicode]) -> None
with logging.pending_warnings():
for docname in status_iterator(docnames, 'writing output... ', "darkgreen",
len(docnames), self.app.verbosity):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
self.write_doc(docname, doctree)
def finish(self):
self.env.get_doctree = self._original_get_doctree
if self.publish:
self.legacy_assets = {}
self.legacy_pages = None
self.parent_id = self.publisher.getBasePageId()
for docname in status_iterator(
self.publish_docnames, 'publishing documents... ',
length=len(self.publish_docnames),
verbosity=self.app.verbosity):
if self.publish_subset and docname not in self.publish_subset:
continue
docfile = path.join(self.outdir, self.file_transform(docname))
try:
with io.open(docfile, 'r', encoding='utf-8') as file:
output = file.read()
self.publish_doc(docname, output)
except (IOError, OSError) as err:
ConfluenceLogger.warn("error reading file %s: "
"%s" % (docfile, err))
def to_asset_name(asset):
return asset[0]
assets = self.assets.build()
for asset in status_iterator(assets, 'publishing assets... ',
length=len(assets), verbosity=self.app.verbosity,
stringify_func=to_asset_name):
key, absfile, type, hash, docname = asset
if self.publish_subset and docname not in self.publish_subset:
continue
try:
with open(absfile, 'rb') as file:
output = file.read()
self.publish_asset(key, docname, output, type, hash)
except (IOError, OSError) as err:
ConfluenceLogger.warn("error reading asset %s: "
"%s" % (key, err))
self.publish_purge()
self.publish_finalize()
def write(self, build_docnames, updated_docnames, method='update'):
for docname in status_iterator(sorted(build_docnames),
'running lint... ',
length=len(build_docnames)):
self.check_doc(docname, self.env.get_doctree(docname))
for warning in self.warnings:
self.warn(warning)
def _read_serial(self, docnames, app):
# type: (List[unicode], Sphinx) -> None
for docname in status_iterator(docnames, 'reading sources... ', "purple",
len(docnames), self.app.verbosity):
# remove all inventory entries for that file
app.emit('env-purge-doc', self, docname)
self.clear_doc(docname)
self.read_doc(docname, app)
def finish(self):
self.env.get_doctree = self._original_get_doctree
if self.publish:
self.legacy_assets = {}
self.legacy_pages = None
self.parent_id = self.publisher.getBasePageId()
for docname in status_iterator(
self.publish_docnames, 'publishing documents... ',
length=len(self.publish_docnames),
verbosity=self.app.verbosity):
if self.publish_subset and docname not in self.publish_subset:
continue
docfile = path.join(self.outdir, self.file_transform(docname))
try:
with io.open(docfile, 'r', encoding='utf-8') as file:
output = file.read()
self.publish_doc(docname, output)
except (IOError, OSError) as err:
ConfluenceLogger.warn("error reading file %s: "
"%s" % (docfile, err))
def to_asset_name(asset):
return asset[0]
assets = self.assets.build()
for asset in status_iterator(assets, 'publishing assets... ',
length=len(assets), verbosity=self.app.verbosity,
stringify_func=to_asset_name):
key, absfile, type, hash, docname = asset
if self.publish_subset and docname not in self.publish_subset:
continue
try:
with open(absfile, 'rb') as file:
output = file.read()
self.publish_asset(key, docname, output, type, hash)
except (IOError, OSError) as err:
ConfluenceLogger.warn("error reading asset %s: "
"%s" % (key, err))
self.publish_purge()
self.publish_finalize()
def _write_serial(self, docnames):
# type: (Sequence[unicode]) -> None
with logging.pending_warnings():
for docname in status_iterator(docnames, 'writing output... ', "darkgreen",
len(docnames), self.app.verbosity):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
self.write_doc(docname, doctree)
def status_iterator(self, elements, message):
return status_iterator(
elements,
message,
"purple",
length=len(elements),
verbosity=self.sphinx_env.app.verbosity,
)
def find_copy_make_toc(type_, docs_dir, search_root, header):
"""Look for documentation files, copy them to the build directory, and
generate toc file linking to pod/site documentation.
Args:
type_ (str): either "pod" or "site".
docs_dir (str): Directory this script is located in.
search_root (str): Directory to search for PODs or sites.
header (str): header of the toc file.
"""
def _docname(item):
"""Helper for status_iterator()."""
return str(os.path.basename(item))
# destination directory to copy docs to
sphinx_subdir = 'sphinx_{}s'.format(type_)
sphinx_dir = os.path.join(docs_dir, sphinx_subdir)
if not os.path.isdir(sphinx_dir):
os.makedirs(sphinx_dir)
# find PODs or sites as directories under search_root
entries = [x for x in os.listdir(search_root) \
if os.path.isdir(os.path.join(search_root, x)) and x[0].isalnum()
]
# Case-insensitive alpha sort
entries = sorted(entries, key=(lambda s: s.lower())) # handles unicode
if 'example' in entries:
# put example POD documentation first
entries.remove('example')
entries.insert(0, 'example')
# find documentation files
# = all non-PDF files (.rst and graphics) in /doc subdirectory
docs = []
for entry in entries:
doc_dir = os.path.join(search_root, entry, 'doc')
if os.path.isdir(doc_dir):
docs.extend([
os.path.join(doc_dir, x) for x in os.listdir(doc_dir) \
if os.path.isfile(os.path.join(doc_dir, x)) and not x.endswith('.pdf')
])
# copy the docs we found
iter_ = status_iterator(docs,
'Copying {} files... '.format(type_),
color='purple',
stringify_func=_docname)
for source in iter_:
shutil.copy2(source, sphinx_dir)
# create toc file, either "pod_toc.rst" or "site_toc.rst"
toc_path = os.path.join(docs_dir, 'sphinx', '{}_toc.rst'.format(type_))
if os.path.exists(toc_path):
os.remove(toc_path)
with open(toc_path, 'w') as file_:
file_.write(header)
for entry in entries:
# correct number of spaces for .rst indentation
file_.write(' ../{}/{}\n'.format(sphinx_subdir, entry))
def status_iterator(self, iterable, summary, colorfunc=darkgreen, length=0,
stringify_func=_display_chunk):
# type: (Iterable, unicode, Callable, int, Callable[[Any], unicode]) -> Iterable
warnings.warn('app.status_iterator() is now deprecated. '
'Use sphinx.util.status_iterator() instead.',
RemovedInSphinx17Warning)
for item in status_iterator(iterable, summary, length=length, verbosity=self.verbosity,
color="darkgreen", stringify_func=stringify_func):
yield item
def status_iterator(self, iterable, summary, colorfunc=darkgreen, length=0,
stringify_func=_display_chunk):
# type: (Iterable, unicode, Callable, int, Callable[[Any], unicode]) -> Iterable
warnings.warn('app.status_iterator() is now deprecated. '
'Use sphinx.util.status_iterator() instead.',
RemovedInSphinx17Warning)
for item in status_iterator(iterable, summary, length=length, verbosity=self.verbosity,
color="darkgreen", stringify_func=stringify_func):
yield item
def _parse(self):
# 1. pull in global configuration from conf.py
self.sections = self.env.config.changelog_sections
self.inner_tag_sort = self.env.config.changelog_inner_tag_sort + [""]
# 2. examine top level directives inside the .. changelog::
# directive. version, release date
self._parsed_content = parsed = _parse_content(self.content)
self.version = version = parsed.get('version', '')
self.release_date = parsed.get('released', None)
self.is_released = bool(self.release_date)
self.env.temp_data['ChangeLogDirective'] = self
content = self.content
# 3. read extra per-file included notes
if 'include_notes_from' in parsed:
if content.items and content.items[0]:
source = content.items[0][0]
path = os.path.join(os.path.dirname(source),
parsed['include_notes_from'])
else:
path = parsed['include_notes_from']
if not os.path.exists(path):
raise Exception("included nodes path %s does not exist" % path)
content = copy.deepcopy(content)
files = [
fname for fname in os.listdir(path) if fname.endswith(".rst")
]
for fname in status_iterator(
files,
"reading changelog note files (version %s)..." % version,
"purple",
length=len(files),
verbosity=self.env.app.verbosity):
fpath = os.path.join(path, fname)
with open(fpath) as handle:
content.append("", path, 0)
for num, line in enumerate(handle):
if "\t" in line:
warnings.warn("file %s has a tab in it! please "
"convert to spaces." % fname)
line = line.replace("\t", " ")
line = line.rstrip()
content.append(line, path, num)
# 4. parse the content of the .. changelog:: directive. This
# is where we parse individual .. change:: directives and construct
# a list of items, stored in the env via self.get_changes_list(env)
p = nodes.paragraph(
'',
'',
)
self.state.nested_parse(content[1:], 0, p)
def _write_serial(self, docnames: Sequence[str]) -> None:
with logging.pending_warnings():
for docname in status_iterator(docnames, __('writing output... '), "darkgreen",
len(docnames), self.app.verbosity):
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(docname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(docname, doctree)
self.write_doc(docname, doctree)
def _write_serial(self, docnames):
# type: (Sequence[str]) -> None
with logging.pending_warnings():
for docname in status_iterator(docnames, __('writing output... '), "darkgreen",
len(docnames), self.app.verbosity):
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(docname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(docname, doctree)
self.write_doc(docname, doctree)
def load(self):
"""
Read the files associated with the sls objects and parse their comment blocks
"""
# Process all the sls objects and their files
for sls_obj in status_iterator(
self.sls_objects,
bold("[AutoSaltSLS] Reading primary sls... "),
"darkgreen",
self.sls_objects_count,
1,
stringify_func=_stringify_sls,
):
# Parse the sls object's file and add to the object as an entry
sls_obj.parse_file()
# Some debugging info
if sls_obj.header.has_text:
logger.debug("[AutoSaltSLS] {0} extracted header:\n{1}".format(
"Top File" if sls_obj.topfile else "File",
sls_obj.header.annotated_text,
))
if sls_obj.body:
logger.debug("[AutoSaltSLS] {0} extracted body:\n{1}".format(
"Top File" if sls_obj.topfile else "File",
sls_obj.annotated_body,
))
# Now parse any files belong to its children
for sls_child_obj in status_iterator(
sls_obj.children,
bold("[AutoSaltSLS] Reading child sls... "),
"darkgreen",
sls_obj.child_count,
1,
stringify_func=_stringify_sls,
):
sls_child_obj.parse_file()
if sls_child_obj.text:
logger.debug(
"[AutoSaltSLS] Child extracted text:\n{0}".format(
sls_child_obj.text))
def run(self):
env = self.state.document.settings.env
app = env.app
# workaround (used below) for https://github.com/sphinx-doc/sphinx/issues/3924
current_docname = env.docname
docdir = dirname(env.doc2path(env.docname))
specpath = join(docdir, self.arguments[0])
dest_dir = join(dirname(specpath), "gallery")
ensuredir(dest_dir)
env.note_dependency(specpath)
spec = json.load(open(specpath))
details = spec['details']
details_iter = status_iterator(
details,
'copying gallery files... ',
'brown',
len(details),
stringify_func=lambda x: x['name'] + ".py")
env.gallery_updated = []
for detail in details_iter:
src_path = abspath(join("..", detail['path']))
dest_path = join(dest_dir, detail['name'] + ".py")
# sphinx pickled env works only with forward slash
docname = join(env.app.config.bokeh_gallery_dir,
detail['name']).replace("\\", "/")
try:
copyfile(src_path, dest_path)
except OSError as e:
raise SphinxError('cannot copy gallery file %r, reason: %s' %
(src_path, e))
try:
env.clear_doc(docname)
env.read_doc(docname, app=app)
env.gallery_updated.append(docname)
except Exception as e:
raise SphinxError('failed to read gallery doc %r, reason: %s' %
(docname, e))
names = [detail['name'] for detail in details]
rst_text = GALLERY_PAGE.render(names=names)
# workaround for https://github.com/sphinx-doc/sphinx/issues/3924
env.temp_data['docname'] = current_docname
return self._parse(rst_text, "<bokeh-gallery>")
def finish(self):
# type: () -> None
super(MessageCatalogBuilder, self).finish()
data = {
'version':
self.config.version,
'copyright':
self.config.copyright,
'project':
self.config.project,
'ctime':
datetime.fromtimestamp(timestamp,
ltz).strftime('%Y-%m-%d %H:%M%z'),
}
for textdomain, catalog in status_iterator(
self.catalogs.items(), # type: ignore
__("writing message catalogs... "),
"darkgreen",
len(self.catalogs),
self.app.verbosity,
lambda textdomain__: textdomain__[0]):
# noop if config.gettext_compact is set
ensuredir(path.join(self.outdir, path.dirname(textdomain)))
pofn = path.join(self.outdir, textdomain + '.pot')
output = StringIO()
output.write(POHEADER % data) # type: ignore
for message in catalog.messages:
positions = catalog.metadata[message]
if self.config.gettext_location:
# generate "#: file1:line1\n#: file2:line2 ..."
output.write("#: %s\n" % "\n#: ".join( # type: ignore
"%s:%s" %
(canon_path(relpath(source, self.outdir)), line)
for source, line, _ in positions))
if self.config.gettext_uuid:
# generate "# uuid1\n# uuid2\n ..."
output.write("# %s\n" % "\n# ".join( # type: ignore
uid for _, _, uid in positions))
# message contains *one* line of text ready for translation
message = message.replace('\\', r'\\'). \
replace('"', r'\"'). \
replace('\n', '\\n"\n"')
output.write('msgid "%s"\nmsgstr ""\n\n' %
message) # type: ignore
content = output.getvalue()
if should_write(pofn, content):
with open(pofn, 'w',
encoding='utf-8') as pofile: # type: ignore
pofile.write(content)
def finish(self) -> None:
# Bypass MessageCatalogBuilder.finish
I18nBuilder.finish(self)
# This is mostly copy pasted from Sphinx
# However, this allows
context = {
'version':
self.config.version,
'copyright':
self.config.copyright,
'project':
self.config.project,
'last_translator':
self.config.gettext_last_translator,
'language_team':
self.config.gettext_language_team,
'ctime':
datetime.datetime.fromtimestamp(timestamp,
ltz).strftime('%Y-%m-%d %H:%M%z'),
'display_location':
self.config.gettext_location,
'display_uuid':
self.config.gettext_uuid,
}
REGEX = self._ADMONITION_REGEX
for textdomain, catalog in status_iterator(
self.catalogs.items(),
__("writing message catalogs... "),
"darkgreen",
len(self.catalogs),
self.app.verbosity,
lambda textdomain__: textdomain__[0],
):
# noop if config.gettext_compact is set
ensuredir(os.path.join(self.outdir, os.path.dirname(textdomain)))
# Due to a bug in Sphinx where messages contain admonitions, this code makes it
# so they're suppressed from the output to prevent the output and CI from breaking
# This is quite a bandaid fix but it seems to work ok
# See https://github.com/sphinx-doc/sphinx/issues/10334
context['messages'] = [
msg for msg in catalog if REGEX.search(msg.text) is None
]
content = GettextRenderer(template_path='_templates/gettext',
outdir=self.outdir).render(
'message.pot_t', context)
pofn = os.path.join(self.outdir, textdomain + '.pot')
if should_write(pofn, content):
with open(pofn, 'w', encoding='utf-8') as pofile:
pofile.write(content)
def install_styles_static_files(app: Sphinx, env):
# Do not copy static_files for our "needs" builder
if app.builder.name == "needs":
return
statics_dir = Path(app.builder.outdir) / IMAGE_DIR_NAME
css_root = Path(__file__).parent / "css"
dest_dir = statics_dir / "sphinx-needs"
def find_css_files() -> Iterable[Path]:
for theme in ["modern", "dark", "blank"]:
if app.config.needs_css == f"{theme}.css":
css_dir = css_root / theme
return [f for f in css_dir.glob("**/*") if f.is_file()]
return [app.config.needs_css]
files_to_copy = [Path("common.css")]
files_to_copy.extend(find_css_files())
# Be sure no "old" css layout is already set
for theme in ["common", "modern", "dark", "blank"]:
path = Path("sphinx-needs") / f"{theme}.css"
safe_remove_file(path, app)
if parse_version(sphinx_version) < parse_version("1.6"):
global status_iterator
status_iterator = app.status_iterator
for source_file_path in status_iterator(
files_to_copy,
"Copying static files for sphinx-needs custom style support...",
brown,
len(files_to_copy),
):
source_file_path = Path(source_file_path)
if not source_file_path.is_absolute():
source_file_path = css_root / source_file_path
if not source_file_path.exists():
source_file_path = css_root / "blank" / "blank.css"
logger.warning(
f"{source_file_path} not found. Copying sphinx-internal blank.css"
)
dest_file = dest_dir / source_file_path.name
dest_dir.mkdir(exist_ok=True)
copyfile(str(source_file_path), str(dest_file))
relative_path = Path(dest_file).relative_to(statics_dir)
safe_add_file(relative_path, app)
def compile_catalogs(self, catalogs: Set[CatalogInfo], message: str) -> None:
if not self.config.gettext_auto_build:
return
def cat2relpath(cat: CatalogInfo) -> str:
return relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
logger.info(bold(__('building [mo]: ')) + message)
for catalog in status_iterator(catalogs, __('writing output... '), "darkgreen",
len(catalogs), self.app.verbosity,
stringify_func=cat2relpath):
catalog.write_mo(self.config.language)
def config_inited_handler(app, config):
gallery_dir = join(app.srcdir, config.bokeh_gallery_dir)
gallery_file = f"{gallery_dir}.json"
if not exists(gallery_file) and isfile(gallery_file):
raise SphinxError(
f"could not find gallery file {gallery_file!r} for configured gallery dir {gallery_dir!r}"
)
gallery_file_mtime = getmtime(gallery_file)
ensuredir(gallery_dir)
# we will remove each file we process from this set and see if anything is
# left at the end (and remove it in that case)
extras = set(os.listdir(gallery_dir))
# app.env.note_dependency(specpath)
spec = json.load(open(gallery_file))
details = spec["details"]
names = {x["name"] for x in details}
if len(names) < len(details):
raise SphinxError(f"gallery file {gallery_file!r} has duplicate names")
details_iter = status_iterator(details,
"creating gallery file entries... ",
"brown",
len(details),
app.verbosity,
stringify_func=lambda x: x["name"] + ".rst")
for detail in details_iter:
detail_file_name = detail["name"] + ".rst"
detail_file_path = join(gallery_dir, detail_file_name)
if detail_file_path in extras:
extras.remove(detail_file_path)
# if the gallery detail file is newer than the gallery file, assume it is up to date
if exists(detail_file_path
) and getmtime(detail_file_path) > gallery_file_mtime:
continue
with open(detail_file_path, "w") as f:
source_path = abspath(join(app.srcdir, "..", "..", detail["path"]))
f.write(
GALLERY_DETAIL.render(filename=detail["name"] + ".py",
source_path=source_path))
for extra_file in extras:
os.remove(join(gallery_dir, extra_file))
def dump(self):
"""Dump store to disk."""
from sphinx.util import status_iterator
# Makesure dir exists
if not path.exists(self.dirname):
os.makedirs(self.dirname)
# Purge orphan items
for key, value in status_iterator(
self._orphan_items.items(),
'purging orphan document(s)... ',
'brown',
len(self._orphan_items),
0,
stringify_func=lambda i: self.stringify(i[0], i[1])):
os.remove(self.itemfile(key))
self.post_purge(key, value)
# Dump dirty items
for key, value in status_iterator(
self._dirty_items.items(),
'dumping dirty document(s)... ',
'brown',
len(self._dirty_items),
0,
stringify_func=lambda i: self.stringify(i[0], i[1])):
with open(self.itemfile(key), 'wb') as f:
pickle.dump(value, f)
self.post_dump(key, value)
# Clear all in-memory items
self._orphan_items = {}
self._dirty_items = {}
self._store = {key: None for key in self._store}
# Dump store itself
with open(self.dictfile(), 'wb') as f:
pickle.dump(self, f)
def write_ncdumps(app):
for name in status_iterator(
app.config.sonatncdumph_ncfiles,
bold("generating list of ncdump -h... "),
length=len(app.config.sonatncdumph_ncfiles),
):
txtfile = app.config.sonatncdumph_filepat.format(name)
txtfile = os.path.join(app.builder.srcdir, txtfile)
checkdir(txtfile, asfile=True)
ncfile = app.config.sonatncdumph_ncfiles[name]
if not os.path.isabs(ncfile):
ncfile = os.path.join(app.builder.srcdir, ncfile)
os.system('ncdump -h {ncfile} > {txtfile}'.format(**locals()))
def run(self):
env = self.state.document.settings.env
app = env.app
# workaround (used below) for https://github.com/sphinx-doc/sphinx/issues/3924
current_docname = env.docname
docdir = dirname(env.doc2path(env.docname))
specpath = join(docdir, self.arguments[0])
dest_dir = join(dirname(specpath), "gallery")
ensuredir(dest_dir)
env.note_dependency(specpath)
spec = json.load(open(specpath))
details = spec['details']
details_iter = status_iterator(details,
'copying gallery files... ',
'brown',
len(details),
stringify_func=lambda x: x['name'] + ".py")
env.gallery_updated = []
for detail in details_iter:
src_path = abspath(join("..", detail['path']))
dest_path = join(dest_dir, detail['name'] + ".py")
# sphinx pickled env works only with forward slash
docname = join(env.app.config.bokeh_gallery_dir, detail['name']).replace("\\","/")
try:
copyfile(src_path, dest_path)
except OSError as e:
raise SphinxError('cannot copy gallery file %r, reason: %s' % (src_path, e))
try:
env.clear_doc(docname)
env.read_doc(docname, app=app)
env.gallery_updated.append(docname)
except Exception as e:
raise SphinxError('failed to read gallery doc %r, reason: %s' % (docname, e))
names = [detail['name']for detail in details]
rst_text = GALLERY_PAGE.render(names=names)
# workaround for https://github.com/sphinx-doc/sphinx/issues/3924
env.temp_data['docname'] = current_docname
return self._parse(rst_text, "<bokeh-gallery>")
def publish_purge(self):
if self.config.confluence_purge:
if self.publish_allowlist or self.publish_denylist:
self.warn(
'confluence_purge disabled due to '
'confluence_publish_allowlist/confluence_publish_denylist')
return
if self.legacy_pages:
for legacy_page_id in status_iterator(
self.legacy_pages,
'removing legacy pages... ',
length=len(self.legacy_pages),
verbosity=self._verbose):
self.publisher.remove_page(legacy_page_id)
# remove any pending assets to remove from the page (as they
# are already been removed)
self.legacy_assets.pop(legacy_page_id, None)
legacy_assets = {}
for legacy_asset_info in self.legacy_assets.values():
for attachment_id, attachment_name in legacy_asset_info.items(
):
legacy_assets[attachment_id] = attachment_name
if legacy_assets:
def to_asset_name(attachment_id):
return legacy_assets[attachment_id]
for attachment_id in status_iterator(
legacy_assets.keys(),
'removing legacy assets... ',
length=len(legacy_assets.keys()),
verbosity=self._verbose,
stringify_func=to_asset_name):
self.publisher.remove_attachment(attachment_id)
def copy_image_files(self):
# type: () -> None
if self.images:
stringify_func = ImageAdapter(self.app.env).get_original_image_uri
for src in status_iterator(self.images, __('copying images... '), "brown",
len(self.images), self.app.verbosity,
stringify_func=stringify_func):
dest = self.images[src]
try:
copy_asset_file(path.join(self.srcdir, src),
path.join(self.outdir, dest))
except Exception as err:
logger.warning(__('cannot copy image file %r: %s'),
path.join(self.srcdir, src), err)
def compile_catalogs(self, catalogs, message):
# type: (Set[CatalogInfo], unicode) -> None
if not self.config.gettext_auto_build:
return
def cat2relpath(cat):
# type: (CatalogInfo) -> unicode
return path.relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
logger.info(bold('building [mo]: ') + message)
for catalog in status_iterator(catalogs, 'writing output... ', "darkgreen",
len(catalogs), self.app.verbosity,
stringify_func=cat2relpath):
catalog.write_mo(self.config.language)
def copy_image_files(self):
# type: () -> None
if self.images:
stringify_func = ImageAdapter(self.app.env).get_original_image_uri
for src in status_iterator(self.images, 'copying images... ', "brown",
len(self.images), self.app.verbosity,
stringify_func=stringify_func):
dest = self.images[src]
try:
copy_asset_file(path.join(self.srcdir, src),
path.join(self.outdir, dest))
except Exception as err:
logger.warning('cannot copy image file %r: %s',
path.join(self.srcdir, src), err)
def compile_catalogs(self, catalogs, message):
# type: (Set[CatalogInfo], unicode) -> None
if not self.config.gettext_auto_build:
return
def cat2relpath(cat):
# type: (CatalogInfo) -> unicode
return relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
logger.info(bold(__('building [mo]: ')) + message)
for catalog in status_iterator(catalogs, __('writing output... '), "darkgreen",
len(catalogs), self.app.verbosity,
stringify_func=cat2relpath):
catalog.write_mo(self.config.language)
def write(self, *ignored):
writer = TextWriter(self)
for label in status_iterator(pydoc_topic_labels,
'building topics... ',
length=len(pydoc_topic_labels)):
if label not in self.env.domaindata['std']['labels']:
self.warn('label %r not in documentation' % label)
continue
docname, labelid, sectname = self.env.domaindata['std']['labels'][label]
doctree = self.env.get_and_resolve_doctree(docname, self)
document = new_document('<section node>')
document.append(doctree.ids[labelid])
destination = StringOutput(encoding='utf-8')
writer.write(document, destination)
self.topics[label] = writer.output
def _extract_from_template(self):
# type: () -> None
files = self._collect_templates()
logger.info(bold('building [%s]: ' % self.name), nonl=1)
logger.info('targets for %d template files', len(files))
extract_translations = self.templates.environment.extract_translations
for template in status_iterator(files, 'reading templates... ', "purple", # type: ignore # NOQA
len(files), self.app.verbosity):
with open(template, 'r', encoding='utf-8') as f: # type: ignore
context = f.read()
for line, meth, msg in extract_translations(context):
origin = MsgOrigin(template, line)
self.catalogs['sphinx'].add(msg, origin)
def config_inited_handler(app, config):
gallery_dir = join(app.srcdir, config.bokeh_gallery_dir)
gallery_file = gallery_dir + ".json"
if not exists(gallery_file) and isfile(gallery_file):
raise SphinxError("could not find gallery file %r for configured gallery dir %r" % (gallery_file, gallery_dir))
gallery_file_mtime = getmtime(gallery_file)
ensuredir(gallery_dir)
# we will remove each file we process from this set and see if anything is
# left at the end (and remove it in that case)
extras = set(os.listdir(gallery_dir))
# app.env.note_dependency(specpath)
spec = json.load(open(gallery_file))
details = spec['details']
names = set(x['name'] for x in details)
if len(names) < len(details):
raise SphinxError("gallery file %r has duplicate names" % gallery_file)
details_iter = status_iterator(details,
'creating gallery file entries... ',
'brown',
len(details),
app.verbosity,
stringify_func=lambda x: x['name'] + ".rst")
for detail in details_iter:
detail_file_name = detail['name'] + ".rst"
detail_file_path = join(gallery_dir, detail_file_name)
if detail_file_path in extras:
extras.remove(detail_file_path)
# if the gallery detail file is newer than the gallery file, assume it is up to date
if exists(detail_file_path) and getmtime(detail_file_path) > gallery_file_mtime:
continue
with open(detail_file_path, "w") as f:
source_path = abspath(join(app.srcdir, "..", "..", detail['path']))
f.write(GALLERY_DETAIL.render(filename=detail['name']+'.py', source_path=source_path))
for extra_file in extras:
os.remove(join(gallery_dir, extra_file))
def finish(self):
# type: () -> None
I18nBuilder.finish(self)
data = dict(
version = self.config.version,
copyright = self.config.copyright,
project = self.config.project,
ctime = datetime.fromtimestamp(
timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'),
)
for textdomain, catalog in status_iterator(iteritems(self.catalogs), # type: ignore
"writing message catalogs... ",
"darkgreen", len(self.catalogs),
self.app.verbosity,
lambda textdomain__: textdomain__[0]):
# noop if config.gettext_compact is set
ensuredir(path.join(self.outdir, path.dirname(textdomain)))
pofn = path.join(self.outdir, textdomain + '.pot')
output = StringIO()
output.write(POHEADER % data) # type: ignore
for message in catalog.messages:
positions = catalog.metadata[message]
if self.config.gettext_location:
# generate "#: file1:line1\n#: file2:line2 ..."
output.write("#: %s\n" % "\n#: ".join( # type: ignore
"%s:%s" % (canon_path(
safe_relpath(source, self.outdir)), line)
for source, line, _ in positions))
if self.config.gettext_uuid:
# generate "# uuid1\n# uuid2\n ..."
output.write("# %s\n" % "\n# ".join( # type: ignore
uid for _, _, uid in positions))
# message contains *one* line of text ready for translation
message = message.replace('\\', r'\\'). \
replace('"', r'\"'). \
replace('\n', '\\n"\n"')
output.write('msgid "%s"\nmsgstr ""\n\n' % message) # type: ignore
content = output.getvalue()
if should_write(pofn, content):
with open(pofn, 'w', encoding='utf-8') as pofile: # type: ignore
pofile.write(content)
def build_finished(app, exception):
""" Generate a ``sitemap.txt`` from the collected HTML page links.
"""
filename = join(app.outdir, "sitemap.txt")
links_iter = status_iterator(sorted(app.sitemap_links),
'adding links to sitemap... ',
'brown',
len(app.sitemap_links))
try:
with open(filename, 'w') as f:
for link in links_iter:
f.write("%s\n" % link)
except OSError as e:
raise SphinxError('cannot write sitemap.txt, reason: %s' % e)
def _extract_from_template(self):
# type: () -> None
files = list(self._collect_templates())
files.sort()
logger.info(bold(__('building [%s]: ') % self.name), nonl=True)
logger.info(__('targets for %d template files'), len(files))
extract_translations = self.templates.environment.extract_translations
for template in status_iterator(files, __('reading templates... '), "purple",
len(files), self.app.verbosity):
try:
with open(template, encoding='utf-8') as f:
context = f.read()
for line, meth, msg in extract_translations(context):
origin = MsgOrigin(template, line)
self.catalogs['sphinx'].add(msg, origin)
except Exception as exc:
raise ThemeError('%s: %r' % (template, exc))
def build_finished(app, exception):
files = set()
for (script, js, js_path, source) in app.env.bokeh_plot_files.values():
files.add(js_path)
files_iter = status_iterator(sorted(files),
'copying bokeh-plot files... ',
'brown',
len(files),
stringify_func=lambda x: basename(x))
for file in files_iter:
target = join(app.builder.outdir, "scripts", basename(file))
ensuredir(dirname(target))
try:
copyfile(file, target)
except OSError as e:
raise SphinxError('cannot copy local file %r, reason: %s' % (file, e))
def copy_image_files_pil(self):
# type: () -> None
"""Copy images using Pillow, the Python Imaging Libary.
The method tries to read and write the files with Pillow, converting
the format and resizing the image if necessary/possible.
"""
ensuredir(path.join(self.outdir, self.imagedir))
for src in status_iterator(self.images, __('copying images... '), "brown",
len(self.images), self.app.verbosity):
dest = self.images[src]
try:
img = Image.open(path.join(self.srcdir, src))
except OSError:
if not self.is_vector_graphics(src):
logger.warning(__('cannot read image file %r: copying it instead'),
path.join(self.srcdir, src))
try:
copyfile(path.join(self.srcdir, src),
path.join(self.outdir, self.imagedir, dest))
except OSError as err:
logger.warning(__('cannot copy image file %r: %s'),
path.join(self.srcdir, src), err)
continue
if self.config.epub_fix_images:
if img.mode in ('P',):
# See the Pillow documentation for Image.convert()
# https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.convert
img = img.convert()
if self.config.epub_max_image_width > 0:
(width, height) = img.size
nw = self.config.epub_max_image_width
if width > nw:
nh = (height * nw) / width
img = img.resize((nw, nh), Image.BICUBIC)
try:
img.save(path.join(self.outdir, self.imagedir, dest))
except OSError as err:
logger.warning(__('cannot write image file %r: %s'),
path.join(self.srcdir, src), err)
def copy_image_files_pil(self):
# type: () -> None
"""Copy images using the PIL.
The method tries to read and write the files with the PIL,
converting the format and resizing the image if necessary/possible.
"""
ensuredir(path.join(self.outdir, self.imagedir))
for src in status_iterator(self.images, 'copying images... ', "brown",
len(self.images), self.app.verbosity):
dest = self.images[src]
try:
img = Image.open(path.join(self.srcdir, src))
except IOError:
if not self.is_vector_graphics(src):
logger.warning('cannot read image file %r: copying it instead',
path.join(self.srcdir, src))
try:
copyfile(path.join(self.srcdir, src),
path.join(self.outdir, self.imagedir, dest))
except (IOError, OSError) as err:
logger.warning('cannot copy image file %r: %s',
path.join(self.srcdir, src), err)
continue
if self.config.epub_fix_images:
if img.mode in ('P',):
# See PIL documentation for Image.convert()
img = img.convert()
if self.config.epub_max_image_width > 0:
(width, height) = img.size
nw = self.config.epub_max_image_width
if width > nw:
nh = (height * nw) / width
img = img.resize((nw, nh), Image.BICUBIC)
try:
img.save(path.join(self.outdir, self.imagedir, dest))
except (IOError, OSError) as err:
logger.warning('cannot write image file %r: %s',
path.join(self.srcdir, src), err)
def _embed_code_links(app, gallery_conf, gallery_dir):
# Add resolvers for the packages for which we want to show links
doc_resolvers = {}
src_gallery_dir = os.path.join(app.builder.srcdir, gallery_dir)
for this_module, url in gallery_conf['reference_url'].items():
try:
if url is None:
doc_resolvers[this_module] = SphinxDocLinkResolver(
app.builder.outdir,
src_gallery_dir,
relative=True)
else:
doc_resolvers[this_module] = SphinxDocLinkResolver(url,
src_gallery_dir)
except HTTPError as e:
print("The following HTTP Error has occurred:\n")
print(e.code)
except URLError as e:
print("\n...\n"
"Warning: Embedding the documentation hyperlinks requires "
"Internet access.\nPlease check your network connection.\n"
"Unable to continue embedding `{0}` links due to a URL "
"Error:\n".format(this_module))
print(e.args)
html_gallery_dir = os.path.abspath(os.path.join(app.builder.outdir,
gallery_dir))
# patterns for replacement
link_pattern = ('<a href="%s" title="View documentation for %s">%s</a>')
orig_pattern = '<span class="n">%s</span>'
period = '<span class="o">.</span>'
# This could be turned into a generator if necessary, but should be okay
flat = [[dirpath, filename]
for dirpath, _, filenames in os.walk(html_gallery_dir)
for filename in filenames]
if LooseVersion(sphinx.__version__) >= LooseVersion('1.6'):
# It will be removed once upgraded to new sphinx-gallery version
from sphinx.util import status_iterator
iterator = status_iterator(
flat, os.path.basename(html_gallery_dir), color='fuchsia',
length=len(flat), stringify_func=lambda x: os.path.basename(x[1]))
else:
iterator = app.status_iterator(
flat, os.path.basename(html_gallery_dir), colorfunc=fuchsia,
length=len(flat), stringify_func=lambda x: os.path.basename(x[1]))
for dirpath, fname in iterator:
full_fname = os.path.join(html_gallery_dir, dirpath, fname)
subpath = dirpath[len(html_gallery_dir) + 1:]
pickle_fname = os.path.join(src_gallery_dir, subpath,
fname[:-5] + '_codeobj.pickle')
if os.path.exists(pickle_fname):
# we have a pickle file with the objects to embed links for
with open(pickle_fname, 'rb') as fid:
example_code_obj = pickle.load(fid)
fid.close()
str_repl = {}
# generate replacement strings with the links
for name, cobj in example_code_obj.items():
this_module = cobj['module'].split('.')[0]
if this_module not in doc_resolvers:
continue
try:
link = doc_resolvers[this_module].resolve(cobj,
full_fname)
except (HTTPError, URLError) as e:
if isinstance(e, HTTPError):
extra = e.code
else:
extra = e.reason
print("\n\t\tError resolving %s.%s: %r (%s)"
% (cobj['module'], cobj['name'], e, extra))
continue
if link is not None:
parts = name.split('.')
name_html = period.join(orig_pattern % part
for part in parts)
full_function_name = '%s.%s' % (
cobj['module'], cobj['name'])
str_repl[name_html] = link_pattern % (
link, full_function_name, name_html)
# do the replacement in the html file
# ensure greediness
names = sorted(str_repl, key=len, reverse=True)
regex_str = '|'.join(re.escape(name) for name in names)
regex = re.compile(regex_str)
def substitute_link(match):
return str_repl[match.group()]
if len(str_repl) > 0:
with open(full_fname, 'rb') as fid:
lines_in = fid.readlines()
with open(full_fname, 'wb') as fid:
for line in lines_in:
line = line.decode('utf-8')
line = regex.sub(substitute_link, line)
fid.write(line.encode('utf-8'))
def generate_recipes(app):
"""
Go through every folder in the `ggd-recipes/recipes` dir,
have a README.rst file generated and generate a recipes.rst from
the collected data.
"""
renderer = Renderer(app)
load_config(os.path.join(os.path.dirname(RECIPE_DIR), "config.yaml"))
repodata = RepoData()
# Add ggd channels to repodata object
#repodata.channels = ['ggd-genomics', 'conda-forge', 'bioconda', 'defaults']
recipes = []
## Get each folder that contains a meat.yaml file
recipe_dirs = []
for root, dirs, files in os.walk(RECIPE_DIR):
if "meta.yaml" in files:
recipe_dirs.append(root)
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(
recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs), app.verbosity):
recipes.extend(generate_readme(folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes = []
for folder in chunk:
_recipes.extend(generate_readme(folder, repodata, renderer))
return _recipes
def merge_chunk(chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
updated = renderer.render_to_file("source/recipes.rst", "recipes.rst_t", {
'recipes': recipes,
# order of columns in the table; must be keys in template_options
'keys': ['Package', 'Version', 'Linux', 'OSX', 'NOARCH'],
'noarch_symbol': '<i class="fa fa-desktop"></i>',
'linux_symbol': '<i class="fa fa-linux"></i>',
'osx_symbol': '<i class="fa fa-apple"></i>',
'dot_symbol': '<i class="fa fa-dot-circle-o"></i>'
})
if updated:
logger.info("Updated source/recipes.rst")
def collect_pages(app):
# type: (Sphinx) -> Iterator[Tuple[unicode, Dict[unicode, Any], unicode]]
env = app.builder.env
if not hasattr(env, '_viewcode_modules'):
return
highlighter = app.builder.highlighter # type: ignore
urito = app.builder.get_relative_uri
modnames = set(env._viewcode_modules) # type: ignore
# app.builder.info(' (%d module code pages)' %
# len(env._viewcode_modules), nonl=1)
for modname, entry in status_iterator(
sorted(env._viewcode_modules.items()), # type: ignore
'highlighting module code... ', "blue",
len(env._viewcode_modules), # type: ignore
app.verbosity, lambda x: x[0]):
if not entry:
continue
code, tags, used, refname = entry
# construct a page name for the highlighted source
pagename = '_modules/' + modname.replace('.', '/')
# highlight the source using the builder's highlighter
if env.config.highlight_language in ('python3', 'default', 'none'):
lexer = env.config.highlight_language
else:
lexer = 'python'
highlighted = highlighter.highlight_block(code, lexer, linenos=False)
# split the code into lines
lines = highlighted.splitlines()
# split off wrap markup from the first line of the actual code
before, after = lines[0].split('<pre>')
lines[0:1] = [before + '<pre>', after]
# nothing to do for the last line; it always starts with </pre> anyway
# now that we have code lines (starting at index 1), insert anchors for
# the collected tags (HACK: this only works if the tag boundaries are
# properly nested!)
maxindex = len(lines) - 1
for name, docname in used.items():
type, start, end = tags[name]
backlink = urito(pagename, docname) + '#' + refname + '.' + name
lines[start] = (
'<div class="viewcode-block" id="%s"><a class="viewcode-back" '
'href="%s">%s</a>' % (name, backlink, _('[docs]')) +
lines[start])
lines[min(end, maxindex)] += '</div>'
# try to find parents (for submodules)
parents = []
parent = modname
while '.' in parent:
parent = parent.rsplit('.', 1)[0]
if parent in modnames:
parents.append({
'link': urito(pagename, '_modules/' +
parent.replace('.', '/')),
'title': parent})
parents.append({'link': urito(pagename, '_modules/index'),
'title': _('Module code')})
parents.reverse()
# putting it all together
context = {
'parents': parents,
'title': modname,
'body': (_('<h1>Source code for %s</h1>') % modname +
'\n'.join(lines)),
} # type: Dict[unicode, Any]
yield (pagename, context, 'page.html')
if not modnames:
return
html = ['\n']
# the stack logic is needed for using nested lists for submodules
stack = ['']
for modname in sorted(modnames):
if modname.startswith(stack[-1]):
stack.append(modname + '.')
html.append('<ul>')
else:
stack.pop()
while not modname.startswith(stack[-1]):
stack.pop()
html.append('</ul>')
stack.append(modname + '.')
html.append('<li><a href="%s">%s</a></li>\n' % (
urito('_modules/index', '_modules/' + modname.replace('.', '/')),
modname))
html.append('</ul>' * (len(stack) - 1))
context = {
'title': _('Overview: module code'),
'body': (_('<h1>All modules for which code is available</h1>') +
''.join(html)),
}
yield ('_modules/index', context, 'page.html')
def gen_gallery(app, doctree):
if app.builder.name != 'html':
return
outdir = app.builder.outdir
rootdir = 'examples'
# Images we want to skip for the gallery because they are an unusual
# size that doesn't layout well in a table, or because they may be
# redundant with other images or uninteresting.
skips = set([
'mathtext_examples',
'matshow_02',
'matshow_03',
'matplotlib_icon',
])
thumbnails = {}
rows = []
random_image = []
toc_rows = []
link_template = ('<a href="{href}">'
'<img src="{thumb_file}" border="0"'
' alt="{alternative_text}"/>'
'</a>')
header_template = ('<div class="section" id="{}">'
'<h4>{}'
'<a class="headerlink" href="#{}"'
' title="Permalink to this headline">¶</a>'
'</h4>')
toc_template = ('<li>'
'<a class="reference internal" href="#{}">{}</a>'
'</li>')
random_image_content_template = '''
// This file was automatically generated by gen_gallery.py & should not be
// modified directly.
images = new Array();
{}
'''
random_image_template = "['{thumbfile}', '{full_image}', '{link}'];"
random_image_join = 'images[{}] = {}'
dirs = ('General', 'Meteorology', 'Oceanography')
for subdir in dirs:
rows.append(header_template.format(subdir, subdir, subdir))
toc_rows.append(toc_template.format(subdir, subdir))
origdir = os.path.join(os.path.dirname(outdir), rootdir, subdir)
if not os.path.exists(origdir):
origdir = os.path.join(os.path.dirname(outdir), 'plot_directive',
rootdir, subdir)
thumbdir = os.path.join(outdir, rootdir, subdir, 'thumbnails')
if not os.path.exists(thumbdir):
os.makedirs(thumbdir)
data = []
for filename in sorted(glob.glob(os.path.join(origdir, '*.png'))):
if filename.endswith('hires.png'):
continue
path, filename = os.path.split(filename)
basename, ext = os.path.splitext(filename)
if basename in skips:
continue
# Create thumbnails based on images in tmpdir, and place them
# within the build tree.
orig_path = str(os.path.join(origdir, filename))
thumb_path = str(os.path.join(thumbdir, filename))
if out_of_date(orig_path, thumb_path) or True:
thumbnails[orig_path] = thumb_path
m = multiimage.match(basename)
if m is not None:
basename = m.group(1)
data.append((subdir, basename,
os.path.join(rootdir, subdir, 'thumbnails',
filename)))
for (subdir, basename, thumbfile) in data:
if thumbfile is not None:
anchor = os.path.basename(thumbfile)
anchor = os.path.splitext(anchor)[0].replace('_', '-')
link = 'examples/{}/{}.html#{}'.format(
subdir,
basename,
anchor)
rows.append(link_template.format(
href=link,
thumb_file=thumbfile,
alternative_text=basename))
random_image.append(random_image_template.format(
link=link,
thumbfile=thumbfile,
basename=basename,
full_image='_images/' + os.path.basename(thumbfile)))
if len(data) == 0:
warnings.warn('No thumbnails were found in {}'.format(subdir))
# Close out the <div> opened up at the top of this loop.
rows.append('</div>')
# Generate JS list of images for front page.
random_image_content = '\n'.join([random_image_join.format(i, line)
for i, line in enumerate(random_image)])
random_image_content = random_image_content_template.format(
random_image_content)
random_image_script_path = os.path.join(app.builder.srcdir,
'_static',
'random_image.js')
with open(random_image_script_path, 'w') as fh:
fh.write(random_image_content)
content = template.format('\n'.join(toc_rows),
'\n'.join(rows))
# Only write out the file if the contents have actually changed.
# Otherwise, this triggers a full rebuild of the docs.
gallery_path = os.path.join(app.builder.srcdir,
'_templates',
'gallery.html')
if os.path.exists(gallery_path):
with open(gallery_path, 'r') as fh:
regenerate = fh.read() != content
else:
regenerate = True
if regenerate:
with open(gallery_path, 'w') as fh:
fh.write(content)
for key in status_iterator(thumbnails,
'generating thumbnails... ',
length=len(thumbnails)):
image.thumbnail(key, thumbnails[key], 0.3)
def generate_recipes(app):
"""Generates recipe RST files
- Checks out repository
- Prepares `RepoData`
- Selects recipes (if BIOCONDA_FILTER_RECIPES in environment)
- Dispatches calls to `generate_readme` for each recipe
- Removes old RST files
"""
source_dir = app.env.srcdir
doctree_dir = app.env.doctreedir # .../build/doctrees
repo_dir = op.join(op.dirname(app.env.srcdir), "_bioconda_recipes")
recipe_basedir = op.join(repo_dir, app.config.bioconda_recipes_path)
repodata_cache_file = op.join(doctree_dir, 'RepoDataCache.pkl')
repo_config_file = os.path.join(repo_dir, app.config.bioconda_config_file)
output_dir = op.join(source_dir, 'recipes')
# Initialize Repo and point globals at the right place
repo = BiocondaRepo(folder=repo_dir, home=app.config.bioconda_repo_url)
repo.checkout_master()
load_config(repo_config_file)
logger.info("Preloading RepoData")
repodata = RepoData()
repodata.set_cache(repodata_cache_file)
repodata.df # pylint: disable=pointless-statement
logger.info("Preloading RepoData (done)")
# Collect recipe names
recipe_dirs = os.listdir(recipe_basedir)
if 'BIOCONDA_FILTER_RECIPES' in os.environ:
limiter = os.environ['BIOCONDA_FILTER_RECIPES']
try:
recipe_dirs = recipe_dirs[:int(limiter)]
except ValueError:
match = re.compile(limiter)
recipe_dirs = [recipe for recipe in recipe_dirs
if match.search(recipe)]
# Set up renderer preparing recipe readme.rst files
recipe_base_url = "{base}/tree/master/{recipes}/".format(
base=app.config.bioconda_repo_url.rstrip(".git"),
recipes=app.config.bioconda_recipes_path
)
renderer = Renderer(app, {'gh_recipes': recipe_base_url})
recipes: List[str] = []
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(
recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs), app.verbosity):
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes: List[Dict[str, Any]] = []
for folder in chunk:
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
_recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
return _recipes
def merge_chunk(_chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
files_wanted = set(recipes)
for root, dirs, files in os.walk(output_dir, topdown=False):
for fname in files:
path = op.join(root, fname)
if path not in files_wanted:
os.unlink(path)
for dname in dirs:
try:
os.rmdir(op.join(root, dname))
except OSError:
pass
