def _check_rst_data(self, data):
"""Return warnings when the provided data has syntax errors."""
source_path = StringIO()
parser = Parser()
settings = frontend.OptionParser().get_default_values()
settings.tab_width = 4
settings.pep_references = None
settings.rfc_references = None
reporter = SilentReporter(
source_path,
settings.report_level,
settings.halt_level,
stream=settings.warning_stream,
debug=settings.debug,
encoding=settings.error_encoding,
error_handler=settings.error_encoding_error_handler)
document = nodes.document(settings, reporter, source=source_path)
document.note_source(source_path, -1)
try:
parser.parse(data, document)
except AttributeError:
reporter.messages.append(
(-1, 'Could not finish the parsing.', '', {}))
return reporter.messages
def renderList(l, markDownHelp, settings=None):
"""
Given a list of reStructuredText or MarkDown sections, return a docutils node list
"""
if len(l) == 0:
return []
if markDownHelp:
from sphinxarg.markdown import parseMarkDownBlock
return parseMarkDownBlock('\n\n'.join(l) + '\n')
else:
all_children = []
for element in l:
if isinstance(element, str):
if settings is None:
settings = OptionParser(
components=(Parser, )).get_default_values()
document = new_document(None, settings)
Parser().parse(element + '\n', document)
all_children += document.children
elif isinstance(element, (nodes.definition, nodes.description)):
all_children += element
else:
assert False, element
return all_children
def _check_rst_data(self, data):
"""Returns warnings when the provided data doesn't compile."""
# the include and csv_table directives need this to be a path
source_path = self.distribution.script_name or 'setup.py'
parser = Parser()
settings = frontend.OptionParser(
components=(Parser, )).get_default_values()
settings.tab_width = 4
settings.pep_references = None
settings.rfc_references = None
reporter = SilentReporter(
source_path,
settings.report_level,
settings.halt_level,
stream=settings.warning_stream,
debug=settings.debug,
encoding=settings.error_encoding,
error_handler=settings.error_encoding_error_handler)
document = nodes.document(settings, reporter, source=source_path)
document.note_source(source_path, -1)
try:
parser.parse(data, document)
except AttributeError as e:
reporter.messages.append(
(-1, 'Could not finish the parsing: %s.' % e, '', {}))
return reporter.messages
def suite():
parser = Parser()
s = DocutilsTestSupport.TransformTestSuite(
parser, suite_settings={'strip_elements_with_classes':
['spam', 'no-ham']})
s.generateTests(totest)
return s
def transform(writer=None, part=None):
p = OptionParser(add_help_option=False)
# Collect all the command line options
docutils_parser = DocutilsOptionParser(components=(writer, Parser()))
for group in docutils_parser.option_groups:
p.add_option_group(group.title, None).add_options(group.option_list)
p.add_option('--part', default=part)
opts, args = p.parse_args()
settings = dict({
'file_insertion_enabled': False,
'raw_enabled': False,
}, **opts.__dict__)
if len(args) == 1:
try:
content = open(args[0], 'r').read()
except IOError:
content = args[0]
else:
content = sys.stdin.read()
parts = publish_parts(
source=content,
settings_overrides=settings,
writer=writer,
)
if opts.part in parts:
return parts[opts.part]
return ''
def get_attr_types(docstring):
parser = Parser()
default_settings = OptionParser(components=(Parser,)).get_default_values()
document = new_document('test_data', default_settings)
parser.parse(docstring, document)
visitor = AttrVisitor(document)
document.walk(visitor)
return visitor.args, visitor.returns
def init_parser(dialect):
if dialect == "rst":
return Parser(), None
if dialect == "fst-rst":
parser = docutils4fstar.LiterateFStarParser()
line_translator = parser.rstline2fstline
return parser, line_translator
assert False
def parse(self, inputstring, document):
pattern = (r'(?:' + TitleParser.symbols + r'+\s)?.+\s' +
TitleParser.symbols +
r'+\s+(?:(?::.+:.*\s)*)((?:.*\s)*?)(?:' +
TitleParser.symbols + r'+\s)?.+\s' + TitleParser.symbols +
r'+')
self.context = re.search(pattern, inputstring).group(1).strip('\n\r')
Parser(self).parse(inputstring, document)
def parseRst(filename):
'''Load a text file and use the .rst parser to build a docutils node tree.'''
settings = OptionParser(components=(Parser, )).get_default_values()
settings.input_encoding = 'utf8' # This does not work, old version of docutils?
source = open(filename).read()
parser = Parser()
doc = docutils.utils.new_document('slides.rst', settings)
parser.parse(source, doc)
return doc
def setup():
global _parser, _settings
_parser = Parser()
_settings = OptionParser().get_default_values()
_settings.tab_width = 8
_settings.pep_references = False
_settings.rfc_references = False
app = FakeSphinx()
sadisp.setup(app)
def parse_rst(rst_string):
parser = Parser()
for name, class_ in []: # Add custom directives here
directives.register(name, class_)
settings = OptionParser(components=(Parser, )).get_default_values()
document = new_document("test", settings)
parser.parse(rst_string, document)
document = parser.document
return document
def auto_code_block(self, node):
"""Try to automatically generate nodes for codeblock syntax.
Parameters
----------
node : nodes.literal_block
Original codeblock node
Returns
-------
tocnode: docutils node
The converted toc tree node, None if conversion is not possible.
"""
assert isinstance(node, nodes.literal_block)
original_node = node
if 'language' not in node:
return None
self.state_machine.reset(self.document, node.parent,
self.current_level)
content = node.rawsource.split('\n')
language = node['language']
if language == 'math':
if self.config['enable_math']:
return self.state_machine.run_directive('math',
content=content)
elif language == 'label':
if self.config['enable_label']:
node = nodes.section()
self.state_machine.state.nested_parse(StringList(
['.. _' + content[0] + ':', ''], source=''),
0,
node=node,
match_titles=True)
return node.children[:]
elif language == 'eval_rst':
if self.config['enable_eval_rst']:
# allow embed non section level rst
node = nodes.section()
self.state_machine.state.nested_parse(StringList(
content, source=original_node.source),
0,
node=node,
match_titles=True)
return node.children[:]
else:
match = re.search('[ ]?[\w_-]+::.*', language)
if match:
parser = Parser()
new_doc = new_document(None, self.document.settings)
newsource = u'.. ' + match.group(0) + '\n' + node.rawsource
parser.parse(newsource, new_doc)
return new_doc.children[:]
else:
return self.state_machine.run_directive('code-block',
arguments=[language],
content=content)
return None
def suite():
parser = Parser()
settings = {'language_code': 'en'}
s = DocutilsTestSupport.TransformTestSuite(parser, suite_settings=settings)
s.generateTests(totest)
settings['language_code'] = 'de'
s.generateTests(totest_de)
settings['language_code'] = 'ru'
s.generateTests(totest_ru)
return s
def _parse_file(self, file: Path):
settings = OptionParser(components=(Parser, )).get_default_values()
parser = Parser()
document = new_document(str(file), settings)
with file.open() as f:
input = f.read()
directives.register_directive("test", TestDirective)
parser.parse(input, document)
return document
def make_citation(label, text, settings):
name = fully_normalize_name(label)
citation = nodes.citation(text)
citation += nodes.label('', label)
new_doc = new_document('temp-string', settings)
parser = Parser()
parser.parse(text, new_doc)
citation['names'].append(name)
citation += new_doc.children
return citation
def suite():
parser = Parser()
settings = {'smart_quotes': True}
s = DocutilsTestSupport.TransformTestSuite(parser, suite_settings=settings)
s.generateTests(totest)
settings['language_code'] = 'de'
s.generateTests(totest_de)
settings['smart_quotes'] = 'alternative'
s.generateTests(totest_de_alt)
return s
def walk(p):
d2 = new_document(
p.pageName(),
OptionParser(components=(Parser, )).get_default_values())
Parser().parse(p.text(), d2)
d += d2.traverse()
for c in page.childrenNesting():
c = p.pageWithName(c)
walk(c)
c.delete()
def _get_document(self, content):
settings = OptionParser(components=(Parser, )).get_default_values()
directives.register_directive("test-block", TestBlockDirective)
directives.register_directive("code-block", CodeBlockDirective)
parser = Parser()
source = str(self.source) if self.source else "lira-unknown-source"
document = new_document(source, settings)
parser.parse(content, document)
return document
def setUp(self):
self.parser = Parser()
self.settings = OptionParser().get_default_values()
self.settings.tab_width = 8
self.settings.pep_references = False
self.settings.rfc_references = False
self.settings.env = Mock(srcdir=os.path.dirname(__file__),
doctreedir=mkdtemp())
self.app = FakeSphinx()
phpautodoc.setup(self.app)
def parse_readme_file(self, formula):
settings = OptionParser(components=(Parser, )).get_default_values()
parser = Parser()
input_file = open('{}/README.rst'.format(formula))
input_data = input_file.read()
document = new_document(input_file.name, settings)
parser.parse(input_data, document)
visitor = SectionParserVisitor(document)
visitor.reset_section_tree()
document.walk(visitor)
input_file.close()
return visitor.get_section_tree()
def run(self, cmd, code):
"""Attempt to parse code as reStructuredText."""
import docutils
from docutils.nodes import Element
from docutils.parsers.rst import Parser
# Generate a new parser
parser = Parser()
settings = docutils.frontend.OptionParser(
components=(docutils.parsers.rst.Parser, )).get_default_values()
document = docutils.utils.new_document(None, settings=settings)
document.reporter.stream = None
document.reporter.halt_level = 5
# Collect errors via an observer
def error_collector(data):
# Mutate the data since it was just generated
data.type = data['type']
data.level = data['level']
data.message = Element.astext(data.children[0])
data.full_message = Element.astext(data)
# Save the error
errors.append(data)
errors = []
document.reporter.attach_observer(error_collector)
parser.parse(code, document)
for data in errors:
message = data.message.replace("\n", " ")
if 'Unknown directive type' in message:
continue
if 'Unknown interpreted text role' in message:
continue
if 'Substitution definition contains illegal element' in message:
# there will be error message for the contents it
# self so let's ignore it.
continue
if data.level >= 3:
error_type = highlight.ERROR
else:
error_type = highlight.WARNING
line = data['line'] - 1
self.highlight.range(line, 0, error_type=error_type)
self.error(line, 0, message, error_type)
def suite():
parser = Parser()
settings = {'smart_quotes': True}
s = DocutilsTestSupport.TransformTestSuite(parser, suite_settings=settings)
s.generateTests(totest)
settings['language_code'] = 'de'
s.generateTests(totest_de)
settings['smart_quotes'] = 'alternative'
s.generateTests(totest_de_alt)
settings['smart_quotes'] = True
settings['smartquotes_locales'] = [('de', u'«»()'), ('nl', u'„”’’')]
s.generateTests(totest_locales)
return s
def parse_doc(dir, file):
parser = Parser()
with open(os.path.join(dir, file + '.rst')) as fh:
doc = new_document(
file,
OptionParser(components=(
docutils.parsers.rst.Parser, )).get_default_values(),
)
parser.parse(
fh.read(),
doc,
)
return doc
def load_rst(filename):
# use the docutils option parser to get defaults for our efforts
opts = OptionParser(components=(Parser, ))
defaults = opts.get_default_values()
# shut up the warnings
defaults.warning_stream = StringIO()
doc = new_document(filename, defaults)
with open(filename) as f:
Parser().parse(f.read(), doc)
return doc
def main():
# process cmdline arguments:
inputFile, outputFile, outputFormat, optargs = getArgs()
settings = OptionParser(components=(Parser, )).get_default_values()
settings.debug = optargs['debug']
parser = Parser()
input = inputFile.read()
document = new_document(inputFile.name, settings)
parser.parse(input, document)
output = format(outputFormat, input, document, optargs)
outputFile.write(output)
if optargs['attributes']:
import pprint
pprint.pprint(document.__dict__)
def renderList(l, markDownHelp, settings=None):
"""
Given a list of reStructuredText or MarkDown sections, return a docutils node list
"""
if len(l) == 0:
return []
if markDownHelp:
return parseMarkDownBlock('\n\n'.join(l) + '\n')
else:
if settings is None:
settings = OptionParser(components=(Parser, )).get_default_values()
document = new_document(None, settings)
Parser().parse('\n\n'.join(l) + '\n', document)
return document.children
def run(self): # check if there are spaces in the notebook name
md_path = self.arguments[0]
if ' ' in md_path:
raise ValueError(
"Due to issues with docutils stripping spaces from links, white "
"space is not allowed in notebook filenames '{0}'".format(
md_path))
# check if raw html is supported
if not self.state.document.settings.raw_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
# get path to markdown file
md_filename = self.arguments[0]
md_dir = os.path.join(setup.confdir, '..')
md_abs_path = os.path.abspath(os.path.join(md_dir, md_filename))
with open(md_abs_path) as file:
source = file.read()
ptype = self.pandoc_from
if 'from' in self.options:
ptype = self.options['from']
if ptype is '':
ptype = 'markdown'
# if ptype != '':
# arglist = ['pandoc', '--from=' + ptype, '--to=rst', md_abs_path]
# else:
# arglist = ['pandoc', '--to=rst', md_abs_path]
#
# p = subprocess.Popen(arglist,
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE
# )
#
# out, err = p.communicate()
#
# print(out)
out = pandoc(source, ptype, 'rst')
settings = OptionParser(components=(Parser, )).get_default_values()
parser = Parser()
document = new_document('DOC', settings)
parser.parse(out, document)
return [node for node in document]
def main(files, dictionary, config):
"""Spell check reStructuredText."""
if not files:
sys.exit(os.EX_USAGE)
if config != DEFAULT_CONFIG and os.path.isfile(config):
print(f"Configuration file '{config}' not found.", file=sys.stderr)
sys.exit(os.EX_NOINPUT)
# ignore Sphinx directives
misspell = Misspell(config)
text_nodes = set([
'block_quote', 'paragraph', 'list_item', 'term',
'definition_list_item', 'title'
])
ignored = ['todo', 'toctree', 'autoclass', 'graphviz', 'automodule']
iroles = ['py:class', 'ref']
for ignore in ignored:
directives.register_directive(ignore, IgnoredDirective)
for role in iroles:
roles.register_local_role(role, ignore_role)
parser = Parser()
settings = OptionParser(components=(Parser, )).get_default_values()
nlp = spacy.load(dictionary)
any_misspellings = False
for file in files:
document = new_document(file, settings)
try:
parser.parse(open(file, 'r').read(), document)
except FileNotFoundError:
print(f"File not found '{file}'", file=sys.stderr)
any_misspellings = True
continue
misspellings = set()
for node in parser.document.traverse(Text):
if (node.tagname == '#text' and node.parent
and node.parent.tagname in text_nodes
and ((node.parent.parent
and node.parent.parent.tagname != 'system_message')
or not node.parent.parent)):
misspellings |= set(token.text for token in nlp(node.astext())
if misspell.is_misspelled(token))
if misspellings:
any_misspellings = True
print(f'✘ {file}')
print(*misspellings, sep='\n')
else:
print(f'✔ {file}')
sys.exit(os.EX_DATAERR if any_misspellings else os.EX_OK)
def process_text(self, input_text):
warning_stream = io.StringIO()
settings_overrides = {}
settings_overrides['warning_stream'] = warning_stream
# Parse the input text using default settings
settings = OptionParser(components=(Parser,)).get_default_values()
parser = Parser()
document = new_document('rstinfo', settings)
parser.parse(input_text, document)
# Transform the parse tree so that the bibliographic data is
# is promoted from a mere field list to a `docinfo` node
t = Transformer(document)
t.add_transforms([frontmatter.DocTitle, frontmatter.DocInfo])
t.apply_transforms()
info = {}
# Process individual nodes which are not part of docinfo.
single_nodes = [
docutils.nodes.title,
docutils.nodes.subtitle,
]
for node in single_nodes:
for doc in document.traverse(node):
if not len(doc.children) == 1:
msg = "Expected node %s to only have 1 child."
raise dexy.exceptions.InternalDexyProblem(msg % node)
info[doc.tagname] = doc.children[0].astext()
# Find the `docinfo` node and extract its children. Non-standard
# bibliographic fields will have the `tagname` 'field' and two
# children, the name and the value. Standard fields simply keep
# the name as the `tagname`.
for doc in document.traverse(docutils.nodes.docinfo):
for element in doc.children:
if element.tagname == 'field':
name, value = element.children
name, value = name.astext(), value.astext()
else:
name, value = element.tagname, element.astext()
info[name] = value
self.log_debug("found info:\n%s\n" % info)
self.update_all_args(info)
self.log_debug("docutils warnings:\n%s\n" % warning_stream.getvalue())
return input_text
def _docutils_cmdline(description, Reader, Parser):
import locale
locale.setlocale(locale.LC_ALL, '')
from docutils.core import publish_cmdline, default_description
from .docutils import setup, HtmlWriter
setup()
parser = Parser()
publish_cmdline(reader=Reader(parser),
parser=parser,
writer=HtmlWriter(),
settings_overrides={'stylesheet_path': None},
description=(description + default_description))
