def extract_summary(doc, document):
# type: (List[unicode], Any) -> unicode
"""Extract summary from docstring."""
# Skip a blank lines at the top
while doc and not doc[0].strip():
doc.pop(0)
# If there's a blank line, then we can assume the first sentence /
# paragraph has ended, so anything after shouldn't be part of the
# summary
for i, piece in enumerate(doc):
if not piece.strip():
doc = doc[:i]
break
# Try to find the "first sentence", which may span multiple lines
sentences = periods_re.split(" ".join(doc)) # type: ignore
if len(sentences) == 1:
summary = sentences[0].strip()
else:
summary = ''
state_machine = RSTStateMachine(state_classes, 'Body')
while sentences:
summary += sentences.pop(0) + '.'
node = new_document('', document.settings)
node.reporter = NullReporter()
state_machine.run([summary], node)
if not node.traverse(nodes.system_message):
# considered as that splitting by period does not break inline markups
break
return summary
def test_extract_summary(capsys):
from sphinx.util.docutils import new_document
from mock import Mock
settings = Mock(language_code='',
id_prefix='',
auto_id_prefix='',
pep_reference=False,
rfc_reference=False)
document = new_document('', settings)
# normal case
doc = ['',
'This is a first sentence. And second one.',
'',
'Second block is here']
assert extract_summary(doc, document) == 'This is a first sentence.'
# inliner case
doc = ['This sentence contains *emphasis text having dots.*,',
'it does not break sentence.']
assert extract_summary(doc, document) == ' '.join(doc)
# abbreviations
doc = ['Blabla, i.e. bla.']
assert extract_summary(doc, document) == 'Blabla, i.e.'
_, err = capsys.readouterr()
assert err == ''
def extract_summary(doc, document):
# type: (List[str], Any) -> str
"""Extract summary from docstring."""
# Skip a blank lines at the top
while doc and not doc[0].strip():
doc.pop(0)
# If there's a blank line, then we can assume the first sentence /
# paragraph has ended, so anything after shouldn't be part of the
# summary
for i, piece in enumerate(doc):
if not piece.strip():
doc = doc[:i]
break
if doc == []:
return ''
# parse the docstring
state_machine = RSTStateMachine(state_classes, 'Body')
node = new_document('', document.settings)
node.reporter = NullReporter()
state_machine.run(doc, node)
if not isinstance(node[0], nodes.paragraph):
# document starts with non-paragraph: pick up the first line
summary = doc[0].strip()
else:
# Try to find the "first sentence", which may span multiple lines
sentences = periods_re.split(" ".join(doc))
if len(sentences) == 1:
summary = sentences[0].strip()
else:
summary = ''
while sentences:
summary += sentences.pop(0) + '.'
node[:] = []
state_machine.run([summary], node)
if not node.traverse(nodes.system_message):
# considered as that splitting by period does not break inline markups
break
# strip literal notation mark ``::`` from tail of summary
summary = literal_re.sub('.', summary)
return summary
def test_RSTParser_prolog_epilog(RSTStateMachine, app):
document = new_document('dummy.rst')
document.settings = Mock(tab_width=8, language_code='')
parser = RSTParser()
parser.set_application(app)
# normal case
text = ('hello Sphinx world\n'
'Sphinx is a document generator')
parser.parse(text, document)
(content, _), _ = RSTStateMachine().run.call_args
assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
('dummy.rst', 1, 'Sphinx is a document generator')]
# with rst_prolog
app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!'
parser.parse(text, document)
(content, _), _ = RSTStateMachine().run.call_args
assert list(content.xitems()) == [('<rst_prolog>', 0, 'this is rst_prolog'),
('<rst_prolog>', 1, 'hello reST!'),
('<generated>', 0, ''),
('dummy.rst', 0, 'hello Sphinx world'),
('dummy.rst', 1, 'Sphinx is a document generator')]
# with rst_epilog
app.env.config.rst_prolog = None
app.env.config.rst_epilog = 'this is rst_epilog\ngood-bye reST!'
parser.parse(text, document)
(content, _), _ = RSTStateMachine().run.call_args
assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
('dummy.rst', 1, 'Sphinx is a document generator'),
('<generated>', 0, ''),
('<rst_epilog>', 0, 'this is rst_epilog'),
('<rst_epilog>', 1, 'good-bye reST!')]
# expandtabs / convert whitespaces
app.env.config.rst_prolog = None
app.env.config.rst_epilog = None
text = ('\thello Sphinx world\n'
'\v\fSphinx is a document generator')
parser.parse(text, document)
(content, _), _ = RSTStateMachine().run.call_args
assert list(content.xitems()) == [('dummy.rst', 0, ' hello Sphinx world'),
('dummy.rst', 1, ' Sphinx is a document generator')]
def apply_transforms(self):
# type: () -> None
if isinstance(self.document, nodes.document):
if not hasattr(self.document.settings, 'env') and self.env:
self.document.settings.env = self.env
super().apply_transforms()
else:
# wrap the target node by document node during transforming
try:
document = new_document('')
if self.env:
document.settings.env = self.env
document += self.document
self.document = document
super().apply_transforms()
finally:
self.document = self.document[0]
def assemble_doctree(self, indexfile, toctree_only, appendices):
# type: (unicode, bool, List[unicode]) -> nodes.Node
from docutils import nodes # NOQA
self.docnames = set([indexfile] + appendices)
logger.info(darkgreen(indexfile) + " ", nonl=1)
tree = self.env.get_doctree(indexfile)
tree['docname'] = indexfile
if toctree_only:
# extract toctree nodes from the tree and put them in a
# fresh document
new_tree = new_document('<latex output>')
new_sect = nodes.section()
new_sect += nodes.title(u'<Set title in conf.py>',
u'<Set title in conf.py>')
new_tree += new_sect
for node in tree.traverse(addnodes.toctree):
new_sect += node
tree = new_tree
largetree = inline_all_toctrees(self, self.docnames, indexfile, tree,
darkgreen, [indexfile])
largetree['docname'] = indexfile
for docname in appendices:
appendix = self.env.get_doctree(docname)
appendix['docname'] = docname
largetree.append(appendix)
logger.info('')
logger.info(__("resolving references..."))
self.env.resolve_references(largetree, indexfile, self)
# resolve :ref:s to distant tex files -- we can't add a cross-reference,
# but append the document name
for pendingnode in largetree.traverse(addnodes.pending_xref):
docname = pendingnode['refdocname']
sectname = pendingnode['refsectname']
newnodes = [nodes.emphasis(sectname, sectname)]
for subdir, title in self.titles:
if docname.startswith(subdir):
newnodes.append(nodes.Text(_(' (in '), _(' (in ')))
newnodes.append(nodes.emphasis(title, title))
newnodes.append(nodes.Text(')', ')'))
break
else:
pass
pendingnode.replace_self(newnodes)
return largetree
def assemble_doctree(self, indexfile: str, toctree_only: bool,
appendices: List[str]) -> nodes.document: # NOQA
self.docnames = set([indexfile] + appendices)
logger.info(darkgreen(indexfile) + " ", nonl=True)
tree = self.env.get_doctree(indexfile)
tree['docname'] = indexfile
if toctree_only:
# extract toctree nodes from the tree and put them in a
# fresh document
new_tree = new_document('<latex output>')
new_sect = nodes.section()
new_sect += nodes.title('<Set title in conf.py>',
'<Set title in conf.py>')
new_tree += new_sect
for node in tree.traverse(addnodes.toctree):
new_sect += node
tree = new_tree
largetree = inline_all_toctrees(self, self.docnames, indexfile, tree,
darkgreen, [indexfile])
largetree['docname'] = indexfile
for docname in appendices:
appendix = self.env.get_doctree(docname)
appendix['docname'] = docname
largetree.append(appendix)
logger.info('')
logger.info(__("resolving references..."))
self.env.resolve_references(largetree, indexfile, self)
# resolve :ref:s to distant tex files -- we can't add a cross-reference,
# but append the document name
for pendingnode in largetree.traverse(addnodes.pending_xref):
docname = pendingnode['refdocname']
sectname = pendingnode['refsectname']
newnodes = [nodes.emphasis(sectname, sectname)] # type: List[Node]
for subdir, title in self.titles:
if docname.startswith(subdir):
newnodes.append(nodes.Text(_(' (in '), _(' (in ')))
newnodes.append(nodes.emphasis(title, title))
newnodes.append(nodes.Text(')', ')'))
break
else:
pass
pendingnode.replace_self(newnodes)
return largetree
def assemble_doctree(self, indexfile, toctree_only, appendices):
# type: (unicode, bool, List[unicode]) -> nodes.document
self.docnames = set([indexfile] + appendices)
logger.info(darkgreen(indexfile) + " ", nonl=1)
tree = self.env.get_doctree(indexfile)
tree['docname'] = indexfile
if toctree_only:
# extract toctree nodes from the tree and put them in a
# fresh document
new_tree = new_document('<texinfo output>')
new_sect = nodes.section()
new_sect += nodes.title(u'<Set title in conf.py>',
u'<Set title in conf.py>')
new_tree += new_sect
for node in tree.traverse(addnodes.toctree):
new_sect += node
tree = new_tree
largetree = inline_all_toctrees(self, self.docnames, indexfile, tree,
darkgreen, [indexfile])
largetree['docname'] = indexfile
for docname in appendices:
appendix = self.env.get_doctree(docname)
appendix['docname'] = docname
largetree.append(appendix)
logger.info('')
logger.info(__("resolving references..."))
self.env.resolve_references(largetree, indexfile, self)
# TODO: add support for external :ref:s
for pendingnode in largetree.traverse(addnodes.pending_xref):
docname = pendingnode['refdocname']
sectname = pendingnode['refsectname']
newnodes = [nodes.emphasis(sectname,
sectname)] # type: List[nodes.Node]
for subdir, title in self.titles:
if docname.startswith(subdir):
newnodes.append(nodes.Text(_(' (in '), _(' (in ')))
newnodes.append(nodes.emphasis(title, title))
newnodes.append(nodes.Text(')', ')'))
break
else:
pass
pendingnode.replace_self(newnodes)
return largetree
def assemble_doctree(self, indexfile, toctree_only, appendices):
# type: (str, bool, List[str]) -> nodes.document
self.docnames = set([indexfile] + appendices)
logger.info(darkgreen(indexfile) + " ", nonl=True)
tree = self.env.get_doctree(indexfile)
tree['docname'] = indexfile
if toctree_only:
# extract toctree nodes from the tree and put them in a
# fresh document
new_tree = new_document('<texinfo output>')
new_sect = nodes.section()
new_sect += nodes.title('<Set title in conf.py>',
'<Set title in conf.py>')
new_tree += new_sect
for node in tree.traverse(addnodes.toctree):
new_sect += node
tree = new_tree
largetree = inline_all_toctrees(self, self.docnames, indexfile, tree,
darkgreen, [indexfile])
largetree['docname'] = indexfile
for docname in appendices:
appendix = self.env.get_doctree(docname)
appendix['docname'] = docname
largetree.append(appendix)
logger.info('')
logger.info(__("resolving references..."))
self.env.resolve_references(largetree, indexfile, self)
# TODO: add support for external :ref:s
for pendingnode in largetree.traverse(addnodes.pending_xref):
docname = pendingnode['refdocname']
sectname = pendingnode['refsectname']
newnodes = [nodes.emphasis(sectname, sectname)] # type: List[nodes.Node]
for subdir, title in self.titles:
if docname.startswith(subdir):
newnodes.append(nodes.Text(_(' (in '), _(' (in ')))
newnodes.append(nodes.emphasis(title, title))
newnodes.append(nodes.Text(')', ')'))
break
else:
pass
pendingnode.replace_self(newnodes)
return largetree
def test_extract_summary(capsys):
settings = Mock(language_code='',
id_prefix='',
auto_id_prefix='',
pep_reference=False,
rfc_reference=False)
document = new_document('', settings)
# normal case
doc = ['',
'This is a first sentence. And second one.',
'',
'Second block is here']
assert extract_summary(doc, document) == 'This is a first sentence.'
# inliner case
doc = ['This sentence contains *emphasis text having dots.*,',
'it does not break sentence.']
assert extract_summary(doc, document) == ' '.join(doc)
# abbreviations
doc = ['Blabla, i.e. bla.']
assert extract_summary(doc, document) == ' '.join(doc)
# literal
doc = ['blah blah::']
assert extract_summary(doc, document) == 'blah blah.'
# heading
doc = ['blah blah',
'=========']
assert extract_summary(doc, document) == 'blah blah'
# hyperlink target
doc = ['Do `this <https://www.sphinx-doc.org/>`_ and that. '
'blah blah blah.']
assert (extract_summary(doc, document) ==
'Do `this <https://www.sphinx-doc.org/>`_ and that.')
_, err = capsys.readouterr()
assert err == ''
def test_extract_summary(capsys):
from sphinx.util.docutils import new_document
from mock import Mock
settings = Mock(language_code='',
id_prefix='',
auto_id_prefix='',
pep_reference=False,
rfc_reference=False)
document = new_document('', settings)
# normal case
doc = [
'', 'This is a first sentence. And second one.', '',
'Second block is here'
]
assert extract_summary(doc, document) == 'This is a first sentence.'
# inliner case
doc = [
'This sentence contains *emphasis text having dots.*,',
'it does not break sentence.'
]
assert extract_summary(doc, document) == ' '.join(doc)
# abbreviations
doc = ['Blabla, i.e. bla.']
assert extract_summary(doc, document) == 'Blabla, i.e.'
# literal
doc = ['blah blah::']
assert extract_summary(doc, document) == 'blah blah.'
# heading
doc = ['blah blah', '=========']
assert extract_summary(doc, document) == 'blah blah'
_, err = capsys.readouterr()
assert err == ''
def __init__(self, app):
# After much digging through source code, this is how we emulate Sphinx's builtin reST parsing state
# https://github.com/sphinx-doc/sphinx/blob/68cc0f7e94f360a2c62ebcb761f8096e04ebf07f/sphinx/io.py#L204
# Here we're bypassing the RST Parser, and just doing the relevant code ops
parser = app.registry.create_source_parser(app, "restructuredtext")
# autosummary uses tab width 8; not set by publisher/env for some reason
settings = dict(app.env.settings)
if "tab_width" not in settings:
settings["tab_width"] = 8
p2 = Publisher()
p2.process_programmatic_settings(None, settings, None)
document = new_document('dummy_kissapi_source',
p2.settings) # (source path, settings)
document.reporter = NullReporter()
state_machine = RSTStateMachine(
state_classes, 'Body') # (state machine classes, initial state)
# needed to set various self.[attr] values of state_machine
state_machine.run([""], document) # (input lines, document)
# the directive attrs that are needed
self.state = state_machine.get_state()
self.env = app.env
self.lineno = 0
def test_extract_summary():
from sphinx.util.docutils import new_document
from mock import Mock
settings = Mock(language_code='',
id_prefix='',
auto_id_prefix='',
pep_reference=False,
rfc_reference=False)
document = new_document('', settings)
# normal case
doc = [
'', 'This is a first sentence. And second one.', '',
'Second block is here'
]
assert extract_summary(doc, document) == 'This is a first sentence.'
# inliner case
doc = [
'This sentence contains *emphasis text having dots.*,',
'it does not break sentence.'
]
assert extract_summary(doc, document) == ' '.join(doc)
def extract_summary(doc, document):
# type: (List[unicode], Any) -> unicode
"""Extract summary from docstring."""
# Skip a blank lines at the top
while doc and not doc[0].strip():
doc.pop(0)
# If there's a blank line, then we can assume the first sentence /
# paragraph has ended, so anything after shouldn't be part of the
# summary
for i, piece in enumerate(doc):
if not piece.strip():
doc = doc[:i]
break
# Try to find the "first sentence", which may span multiple lines
sentences = periods_re.split(" ".join(doc)) # type: ignore
if len(sentences) == 1:
summary = sentences[0].strip()
else:
summary = ''
state_machine = RSTStateMachine(state_classes, 'Body')
while sentences:
summary += sentences.pop(0) + '.'
node = new_document('', document.settings)
node.reporter = NullReporter()
state_machine.run([summary], node)
if not node.traverse(nodes.system_message):
# considered as that splitting by period does not break inline markups
break
# strip literal notation mark ``::`` from tail of summary
summary = literal_re.sub('.', summary)
return summary
def process_solutions(self, doctree: nodes.document, src: str) -> None:
"""Handle any solutions contained in the document.
This ensures that a ``*.py`` file is created in the ``resources`` directory
containing the actual solution.
It then also rewrites the given doctree to output a pair of code cells in
the resulting notebook. The first is a prompt for the user to input their
solution and the second contains a :magic:`ipython:load` declaration to
give the user the option to load in the solution if they wish to see it.
Parameters
----------
doctree:
The doctree to process
src:
The path to the file containing the document being processed
"""
docpath = pathlib.Path(src)
logger.debug("[tutorial]: processing solutions for: %s", docpath)
basename = f"{docpath.stem}-soln"
for idx, soln in enumerate(doctree.traverse(condition=solution)):
name = f"{basename}-{idx+1:02d}.py"
destination = pathlib.Path("resources", docpath.with_suffix(""), name)
refuri = relative_uri(src, str(destination))
# Convert the solution to a valid Python document that can be executed.
document = new_document("<solution>")
document += soln
# Rather than go through the trouble of maintaining 2 document translators,
# one for notebooks and another for Python files. Let's just use the notebook
# translator and do some post-processing on the result - much easier.
translator = NotebookTranslator(document)
document.walkabout(translator)
notebook = translator.asnotebook()
blocks = []
for cell in notebook.cells:
source = cell.source
# Comment out the lines containing markdown.
if cell.cell_type == "markdown":
source = textwrap.indent(source, "# ")
blocks.append(source)
self.resources[str(destination)] = ("create", "\n".join(blocks))
# TODO: Expose config options for these
# TODO: Translations?
your_soln = nodes.literal_block(
"", "# Write your solution here...\n", language="python"
)
load_soln = nodes.literal_block(
"",
f"# Execute this cell to load the example solution\n%load {refuri}\n",
language="python",
)
# Replace the actual solution with the 2 cells defined above.
soln.children = [your_soln, load_soln]
def process_feed(app, doctree, fromdocname):
env = app.builder.env
document = new_document('')
for node in doctree.traverse(feed):
rss_filename = node['rss']
rss_title = node['title']
rss_link = node['link']
rss_description = node['description']
rss_date = datetime.datetime.utcnow()
rss_items = []
for docname in node['entries']:
entry = env.get_doctree(docname)
for field in entry.traverse(nodes.field):
field_date = [f for f in field.traverse(nodes.field_name)][0]
pars = []
for b in field.traverse(nodes.field_body):
for p in b.traverse(nodes.paragraph):
pars.append(p)
rss_item_description = nodes.compound()
for p in pars[1:]:
rss_item_description += p.deepcopy()
document += rss_item_description
app.env.resolve_references(
document,
node['link'],
app.builder,
)
document.remove(rss_item_description)
if app.builder.format == 'html':
rss_item_description = "\n".join(
app.builder.render_partial(p)['body']
for p in rss_item_description
)
rss_item_date = dateutil.parser.parse(field_date.astext().strip())
rss_item_title = "%s: %s" % (field_date.astext(), pars[0].astext())
rss_item = RSSItem(
rss_item_title,
node['link'],
rss_item_description,
rss_item_date,
)
rss_items.append(rss_item)
node.replace_self([])
if app.builder.format == 'html':
rss_feed = RSSFeed(
rss_title,
rss_link,
rss_description,
rss_date,
rss_items,
)
if rss_filename:
rss_path = os.path.join(app.builder.outdir, rss_filename)
with open(rss_path, 'wb') as rss_stream:
write_rss(rss_feed, rss_stream)
def process_feed(app, doctree, fromdocname):
env = app.builder.env
document = new_document('')
for node in doctree.traverse(feed):
rss_filename = node['rss']
rss_title = node['title']
rss_link = node['link']
rss_description = node['description']
rss_date = datetime.datetime.utcnow()
rss_items = []
for docname in node['entries']:
entry = env.get_doctree(docname)
title = entry.next_node(nodes.section)
first_year = title.next_node(nodes.section)
for year in first_year.traverse(
nodes.section, descend=False, siblings=True
):
first_subsection = year.next_node(nodes.section)
for subsection in first_subsection.traverse(
nodes.section, descend=False, siblings=True
):
if app.builder.format == 'html':
# date
date = subsection.next_node(nodes.title).astext().split(":")[0]
rss_item_date = dateutil.parser.parse(date.strip())
# link
rss_item_link = node["link"] + "#" + subsection['ids'][0]
# title
title = subsection.next_node(nodes.title)
document += title
app.env.resolve_references(
document,
node['link'],
app.builder,
)
document.remove(title)
rss_item_title = title.astext()
# description
pars = []
for b in subsection.traverse(nodes.paragraph):
for p in b.traverse(nodes.paragraph):
pars.append(p)
rss_item_description = nodes.compound()
for p in pars:
rss_item_description += p.deepcopy()
document += rss_item_description
app.env.resolve_references(
document,
node['link'],
app.builder,
)
document.remove(rss_item_description)
rss_item_description = "\n".join(
app.builder.render_partial(p)['body']
for p in rss_item_description
)
if True:
print("--------------------------------------------")
print("--------------------------------------------")
print("--------------------------------------------")
print("newsfeed item:")
print(" title:", rss_item_title)
print(" date:", rss_item_date)
print(" link:", rss_item_link)
print(" desc:", rss_item_description)
print(" ", flush=True)
rss_item = RSSItem(
rss_item_title,
rss_item_link,
rss_item_description,
rss_item_date,
)
rss_items.append(rss_item)
node.replace_self([])
if app.builder.format == 'html':
rss_feed = RSSFeed(
rss_title,
rss_link,
rss_description,
rss_date,
rss_items,
)
if rss_filename:
rss_path = os.path.join(app.builder.outdir, rss_filename)
with open(rss_path, 'wb') as rss_stream:
write_rss(rss_feed, rss_stream)
