def render(self):
key = 'cia.apps.blog.%d' % self.id
parts = cache.get(key)
if not parts:
# Convert the reST markup to a document tree
document = publish_doctree(source=self.content)
visitor = ImageTranslator(document)
document.walkabout(visitor)
#
# Publish that document tree as HTML. We can't use any of
# the simpler methods in docutils.core, since we need
# access to writer.parts
#
reader = doctree.Reader(parser_name='null')
pub = Publisher(reader,
None,
None,
source=DocTreeInput(document),
destination_class=StringOutput)
pub.set_writer('html4css1')
pub.process_programmatic_settings(None, {
'cloak_email_addresses': True,
'initial_header_level': 2,
}, None)
pub.publish()
parts = pub.writer.parts
cache.set(key, parts)
return parts
def get_sphinx():
sphinx = getattr(local_data, 'sphinx', None)
if sphinx is None:
sphinx = Sphinx(tempdir,
tempdir,
tempdir,
tempdir,
'json',
status=None,
warning=None)
sphinx.builder.translator_class = CustomHTMLTranslator
sphinx.env.patch_lookup_functions()
sphinx.env.temp_data['docname'] = 'text'
sphinx.env.temp_data['default_domain'] = 'py'
pub = Publisher(reader=None,
parser=None,
writer=HTMLWriter(sphinx.builder),
source_class=io.StringInput,
destination_class=io.NullOutput)
pub.set_components('standalone', 'restructuredtext', None)
pub.process_programmatic_settings(None, sphinx.env.settings, None)
pub.set_destination(None, None)
sphinx.publisher = pub
local_data.sphinx = sphinx
return sphinx, sphinx.publisher
def get_sphinx():
sphinx = getattr(local_data, 'sphinx', None)
if sphinx is None:
sphinx = Sphinx(tempdir, tempdir, tempdir,
tempdir, 'json', status=None, warning=None)
sphinx.builder.translator_class = CustomHTMLTranslator
sphinx.env.patch_lookup_functions()
sphinx.env.temp_data['docname'] = 'text'
sphinx.env.temp_data['default_domain'] = 'py'
pub = Publisher(reader=None,
parser=None,
writer=HTMLWriter(sphinx.builder),
source_class=io.StringInput,
destination_class=io.NullOutput)
pub.set_components('standalone', 'restructuredtext', None)
pub.process_programmatic_settings(None, sphinx.env.settings, None)
pub.set_destination(None, None)
sphinx.publisher = pub
local_data.sphinx = sphinx
return sphinx, sphinx.publisher
def parse_docstring(doc):
p = Publisher(source=doc, source_class=io.StringInput)
p.set_reader('standalone', p.parser, 'restructuredtext')
p.writer = Writer()
p.process_programmatic_settings(None, None, None)
p.set_source(doc, None)
return p.publish()
def read_doc(app, env, filename):
# type: (Sphinx, BuildEnvironment, unicode) -> nodes.document
"""Parse a document and convert to doctree."""
filetype = get_filetype(app.config.source_suffix, filename)
input_class = app.registry.get_source_input(filetype)
reader = SphinxStandaloneReader(app)
source = input_class(app, env, source=None, source_path=filename,
encoding=env.config.source_encoding)
parser = app.registry.create_source_parser(app, filetype)
if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == ():
# a workaround for recommonmark
# If recommonmark.AutoStrictify is enabled, the parser invokes reST parser
# internally. But recommonmark-0.4.0 does not provide settings_spec for reST
# parser. As a workaround, this copies settings_spec for RSTParser to the
# CommonMarkParser.
parser.settings_spec = RSTParser.settings_spec
pub = Publisher(reader=reader,
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxDummySourceClass,
destination=NullOutput())
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source, filename)
pub.publish()
return pub.document
def read_doc(app: "Sphinx", env: BuildEnvironment,
filename: str) -> nodes.document:
"""Parse a document and convert to doctree."""
# set up error_handler for the target document
error_handler = UnicodeDecodeErrorHandler(env.docname)
codecs.register_error('sphinx', error_handler) # type: ignore
reader = SphinxStandaloneReader()
reader.setup(app)
filetype = get_filetype(app.config.source_suffix, filename)
parser = app.registry.create_source_parser(app, filetype)
if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == (
):
# a workaround for recommonmark
# If recommonmark.AutoStrictify is enabled, the parser invokes reST parser
# internally. But recommonmark-0.4.0 does not provide settings_spec for reST
# parser. As a workaround, this copies settings_spec for RSTParser to the
# CommonMarkParser.
parser.settings_spec = RSTParser.settings_spec
pub = Publisher(reader=reader,
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxFileInput,
destination=NullOutput())
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source_path=filename)
pub.publish()
return pub.document
def read_doc(app, env, filename):
# type: (Sphinx, BuildEnvironment, unicode) -> nodes.document
"""Parse a document and convert to doctree."""
filetype = get_filetype(app.config.source_suffix, filename)
input_class = app.registry.get_source_input(filetype)
reader = SphinxStandaloneReader(app)
source = input_class(app, env, source=None, source_path=filename,
encoding=env.config.source_encoding)
parser = app.registry.create_source_parser(app, filetype)
if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == ():
# a workaround for recommonmark
# If recommonmark.AutoStrictify is enabled, the parser invokes reST parser
# internally. But recommonmark-0.4.0 does not provide settings_spec for reST
# parser. As a workaround, this copies settings_spec for RSTParser to the
# CommonMarkParser.
parser.settings_spec = RSTParser.settings_spec
pub = Publisher(reader=reader,
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxDummySourceClass,
destination=NullOutput())
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source, filename)
pub.publish()
return pub.document
def render(self):
key = 'cia.apps.blog.%d' % self.id
parts = cache.get(key)
if not parts:
# Convert the reST markup to a document tree
document = publish_doctree(source = self.content)
visitor = ImageTranslator(document)
document.walkabout(visitor)
#
# Publish that document tree as HTML. We can't use any of
# the simpler methods in docutils.core, since we need
# access to writer.parts
#
reader = doctree.Reader(parser_name='null')
pub = Publisher(reader, None, None,
source = DocTreeInput(document),
destination_class = StringOutput)
pub.set_writer('html4css1')
pub.process_programmatic_settings(None, {
'cloak_email_addresses': True,
'initial_header_level': 2,
}, None)
pub.publish()
parts = pub.writer.parts
cache.set(key, parts)
return parts
def pypi_render(source):
"""
Copied (and slightly adapted) from pypi.description_tools
"""
ALLOWED_SCHEMES = '''file ftp gopher hdl http https imap mailto mms news
nntp prospero rsync rtsp rtspu sftp shttp sip sips snews svn svn+ssh
telnet wais irc'''.split()
settings_overrides = {
"raw_enabled": 0, # no raw HTML code
"file_insertion_enabled": 0, # no file/URL access
"halt_level": 2, # at warnings or errors, raise an exception
"report_level": 5, # never report problems with the reST code
}
# capture publishing errors, they go to stderr
old_stderr = sys.stderr
sys.stderr = s = StringIO.StringIO()
parts = None
try:
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == '#text':
continue
if node.hasattr('refuri'):
uri = node['refuri']
elif node.hasattr('uri'):
uri = node['uri']
else:
continue
o = urlparse.urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError('link scheme not allowed')
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name='null')
pub = Publisher(reader, source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer('html')
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
except:
pass
sys.stderr = old_stderr
# original text if publishing errors occur
if parts is None or len(s.getvalue()) > 0:
return None
else:
return parts['body']
def process_description(source, output_encoding='unicode'):
"""Given an source string, returns an HTML fragment as a string.
The return value is the contents of the <body> tag.
Parameters:
- `source`: A multi-line text string; required.
- `output_encoding`: The desired encoding of the output. If a Unicode
string is desired, use the default value of "unicode" .
"""
# Dedent all lines of `source`.
source = trim_docstring(source)
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
parts = None
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == '#text':
continue
if node.hasattr('refuri'):
uri = node['refuri']
elif node.hasattr('uri'):
uri = node['uri']
else:
continue
o = urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError('link scheme not allowed: {0}'.format(uri))
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name='null')
pub = Publisher(reader,
source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer('html')
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
output = parts['body']
if output_encoding != 'unicode':
output = output.encode(output_encoding)
return output
def process_description(source, output_encoding='unicode'):
"""Given an source string, returns an HTML fragment as a string.
The return value is the contents of the <body> tag.
Parameters:
- `source`: A multi-line text string; required.
- `output_encoding`: The desired encoding of the output. If a Unicode
string is desired, use the default value of "unicode" .
"""
# Dedent all lines of `source`.
source = trim_docstring(source)
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
parts = None
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == '#text':
continue
if node.hasattr('refuri'):
uri = node['refuri']
elif node.hasattr('uri'):
uri = node['uri']
else:
continue
o = urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError('link scheme not allowed: {0}'.format(uri))
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name='null')
pub = Publisher(reader, source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer('html')
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
output = parts['body']
if output_encoding != 'unicode':
output = output.encode(output_encoding)
return output
def inspect(filename, source_path=None):
"returns the document object before any transforms)"
from docutils.core import Publisher
pub = Publisher(source_class=io.FileInput)
pub.set_reader("standalone", None, "restructuredtext")
pub.process_programmatic_settings(None, None, None)
pub.set_source(source_path=source_path)
pub.set_io()
return pub.reader.read(pub.source, pub.parser, pub.settings)
def __init__(self):
# publisher is used for html generation
pub = Publisher(None, None, None, settings=None,
source_class=StringInput,
destination_class=StringOutput)
pub.set_components('standalone', 'restructuredtext', 'html')
pub.process_programmatic_settings(None, None, None)
pub.set_destination(None, None)
self.pub = pub
def get_html(self, body_only=True, content_only=False, noclasses=False):
import sys
import pygments_rest
from docutils.core import Publisher
from docutils.io import StringInput, StringOutput
from cStringIO import StringIO
settings = {
'doctitle_xform': 1,
'pep_references': 1,
'rfc_references': 1,
'footnote_references': 'superscript',
'output_encoding': 'unicode',
'report_level':
2, # 2=show warnings, 3=show only errors, 5=off (docutils.utils
}
if content_only:
post_rst = self.get_rst(noclasses=noclasses)
else:
post_rst = render_to('post_single.rst',
post=self,
noclasses=noclasses)
pub = Publisher(reader=None,
parser=None,
writer=None,
settings=None,
source_class=StringInput,
destination_class=StringOutput)
pub.set_components(reader_name='standalone',
parser_name='restructuredtext',
writer_name='html')
pub.process_programmatic_settings(settings_spec=None,
settings_overrides=settings,
config_section=None)
pub.set_source(post_rst, source_path=self.module_path)
pub.set_destination(None, None)
errors_io = StringIO()
real_stderr = sys.stderr
sys.stderr = errors_io
try:
html_full = pub.publish(enable_exit_status=False)
html_body = ''.join(pub.writer.html_body)
finally:
sys.stderr = real_stderr
errors = errors_io.getvalue()
self._process_rest_errors(errors)
errors_io.close()
return html_body if body_only else html_full
def inspect(
filename,
source_path=None,
):
"returns the document object before any transforms)"
from docutils.core import Publisher
pub = Publisher(source_class=io.FileInput, )
pub.set_reader('standalone', None, "restructuredtext")
pub.process_programmatic_settings(None, None, None)
pub.set_source(source_path=source_path)
pub.set_io()
return pub.reader.read(pub.source, pub.parser, pub.settings)
def convert(infilename, outfilename):
print "converting %s to %s" % (infilename, outfilename)
pub = Publisher()
pub.set_components('standalone', # reader
'restructuredtext', # parser
'latex') # writer (arg, will be discarded)
pub.reader = StandaloneReader()
pub.writer = VerseLaTeXWriter()
pub.process_programmatic_settings(None, None, None)
pub.set_source(source_path=infilename)
pub.set_destination(destination_path=outfilename)
pub.publish()
def get_html(self, body_only=True, content_only=False, noclasses=False):
import sys
import pygments_rest
from docutils.core import Publisher
from docutils.io import StringInput, StringOutput
from cStringIO import StringIO
settings = {'doctitle_xform' : 1,
'pep_references' : 1,
'rfc_references' : 1,
'footnote_references': 'superscript',
'output_encoding' : 'unicode',
'report_level' : 2, # 2=show warnings, 3=show only errors, 5=off (docutils.utils
}
if content_only:
post_rst = self.get_rst(noclasses=noclasses)
else:
post_rst = render_to('post_single.rst',
post=self,
noclasses=noclasses)
pub = Publisher(reader=None,
parser=None,
writer=None,
settings=None,
source_class=StringInput,
destination_class=StringOutput)
pub.set_components(reader_name='standalone',
parser_name='restructuredtext',
writer_name='html')
pub.process_programmatic_settings(settings_spec=None,
settings_overrides=settings,
config_section=None)
pub.set_source(post_rst,source_path=self.module_path)
pub.set_destination(None, None)
errors_io = StringIO()
real_stderr = sys.stderr
sys.stderr = errors_io
try:
html_full = pub.publish(enable_exit_status=False)
html_body = ''.join(pub.writer.html_body)
finally:
sys.stderr = real_stderr
errors = errors_io.getvalue()
self._process_rest_errors(errors)
errors_io.close()
return html_body if body_only else html_full
def convert(infilename, outfilename):
print "converting %s to %s" % (infilename, outfilename)
pub = Publisher()
pub.set_components(
'standalone', # reader
'restructuredtext', # parser
'latex') # writer (arg, will be discarded)
pub.reader = OptikReader()
pub.writer = PyLaTeXWriter()
pub.process_programmatic_settings(None, None, None)
pub.set_source(source_path=infilename)
pub.set_destination(destination_path=outfilename)
pub.publish()
def process_labels(site, logger, source, post):
site.processing_labels = True
pub = Publisher(reader=Reader(), parser=None, writer=None)
pub.set_components(None, 'restructuredtext', 'html')
# Reading the file will generate output/errors that we don't care about
# at this stage. The report_level = 5 means no output
pub.process_programmatic_settings(
settings_spec=None,
settings_overrides={'report_level': 5},
config_section=None,
)
pub.set_source(None, source)
pub.publish()
document = pub.document
site.processing_labels = False
# Code based on Sphinx std domain
for name, is_explicit in document.nametypes.items():
if not is_explicit:
continue
labelid = document.nameids[name]
if labelid is None:
continue
node = document.ids[labelid]
if node.tagname == 'target' and 'refid' in node:
node = document.ids.get(node['refid'])
labelid = node['names'][0]
if node.tagname == 'footnote' or 'refuri' in node or node.tagname.startswith(
'desc_'):
continue
if name in site.ref_labels:
logger.warn(
'Duplicate label {dup}, other instance in {other}'.format(
dup=name, other=site.ref_labels[name][0]))
site.anon_ref_labels[name] = post.permalink(), labelid
def clean_astext(node):
"""Like node.astext(), but ignore images.
Taken from sphinx.util.nodes"""
node = node.deepcopy()
for img in node.traverse(nodes.image):
img['alt'] = ''
for raw in node.traverse(nodes.raw):
raw.parent.remove(raw)
return node.astext()
if node.tagname in ('section', 'rubric'):
sectname = clean_astext(node[0])
else:
continue
site.ref_labels[name] = post.permalink(), labelid, sectname
class HTMLGenerator:
"""
Really simple HTMLGenerator starting from publish_parts
It reuses the docutils.core.Publisher class, which means it is *not*
threadsafe.
"""
def __init__(self,
settings_spec=None,
settings_overrides=None,
config_section='general'):
if settings_overrides is None:
settings_overrides = {'report_level': 5, 'halt_level': 5}
self.pub = Publisher(reader=None,
parser=None,
writer=None,
settings=None,
source_class=io.StringInput,
destination_class=io.StringOutput)
self.pub.set_components(reader_name='standalone',
parser_name='restructuredtext',
writer_name='html')
# hack: JEP-0071 does not allow HTML char entities, so we hack our way
# out of it.
# — == u"\u2014"
# a setting to only emit charater entities in the writer would be nice
# FIXME: several are emitted, and they are explicitly forbidden
# in the JEP
# == u"\u00a0"
self.pub.writer.translator_class.attribution_formats['dash'] = (
'\u2014', '')
self.pub.process_programmatic_settings(settings_spec,
settings_overrides,
config_section)
def create_xhtml(self,
text,
destination=None,
destination_path=None,
enable_exit_status=None):
"""
Create xhtml for a fragment of IM dialog. We can use the source_name
to store info about the message
"""
self.pub.set_source(text, None)
self.pub.set_destination(destination, destination_path)
output = self.pub.publish(enable_exit_status=enable_exit_status)
# kludge until we can get docutils to stop generating (rare)
# entities
return '\u00a0'.join(
self.pub.writer.parts['fragment'].strip().split(' '))
def render_readme_like_pypi(source, output_encoding='unicode'):
"""
Render a ReST document just like PyPI does.
"""
# Dedent all lines of `source`.
source = trim_docstring(source)
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
parts = None
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == '#text':
continue
if node.hasattr('refuri'):
uri = node['refuri']
elif node.hasattr('uri'):
uri = node['uri']
else:
continue
o = urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError('link scheme not allowed: {0}'.format(uri))
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name='null')
pub = Publisher(reader,
source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer('html')
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
output = parts['body']
if output_encoding != 'unicode':
output = output.encode(output_encoding)
return output
def render_readme_like_pypi(source, output_encoding='unicode'):
"""
Render a ReST document just like PyPI does.
"""
# Dedent all lines of `source`.
source = trim_docstring(source)
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
parts = None
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == '#text':
continue
if node.hasattr('refuri'):
uri = node['refuri']
elif node.hasattr('uri'):
uri = node['uri']
else:
continue
o = urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError('link scheme not allowed: {0}'.format(uri))
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name='null')
pub = Publisher(reader, source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer('html')
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
output = parts['body']
if output_encoding != 'unicode':
output = output.encode(output_encoding)
return output
def _get_publisher(self, source_path):
extra_params = {'initial_header_level': '2',
'syntax_highlight': 'short',
'input_encoding': 'utf-8'}
user_params = self.settings.get('DOCUTILS_SETTINGS')
if user_params:
extra_params.update(user_params)
pub = Publisher(destination_class=StringOutput)
pub.set_components('standalone', 'restructuredtext', 'html')
pub.writer.translator_class = HTMLTranslator
pub.process_programmatic_settings(None, extra_params, None)
pub.set_source(source_path=source_path)
pub.publish()
return pub
def get_doctree(path):
"""
Obtain a Sphinx doctree from the RST file at ``path``.
Performs no Releases-specific processing; this code would, ideally, be in
Sphinx itself, but things there are pretty tightly coupled. So we wrote
this.
:param str path: A relative or absolute file path string.
:returns:
A two-tuple of the generated ``sphinx.application.Sphinx`` app and the
doctree (a ``docutils.document`` object).
"""
root, filename = os.path.split(path)
docname, _ = os.path.splitext(filename)
# TODO: this only works for top level changelog files (i.e. ones where
# their dirname is the project/doc root)
app = make_app(srcdir=root)
# Create & init a BuildEnvironment. Mm, tasty side effects.
app._init_env(freshenv=True)
env = app.env
env.update(config=app.config, srcdir=root, doctreedir=app.doctreedir, app=app)
# Code taken from sphinx.environment.read_doc; easier to manually call
# it with a working Environment object, instead of doing more random crap
# to trick the higher up build system into thinking our single changelog
# document was "updated".
env.temp_data['docname'] = docname
env.app = app
# NOTE: SphinxStandaloneReader API changed in 1.4 :(
reader_kwargs = {'app': env.app, 'parsers': env.config.source_parsers}
if sphinx.version_info[:2] < (1, 4):
del reader_kwargs['app']
reader = SphinxStandaloneReader(**reader_kwargs)
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
# NOTE: docname derived higher up, from our given path
src_path = env.doc2path(docname)
source = SphinxFileInput(app, env, source=None, source_path=src_path,
encoding=env.config.source_encoding)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
return app, pub.document
def read_doc(app, env, filename):
# type: (Sphinx, BuildEnvironment, unicode) -> nodes.document
"""Parse a document and convert to doctree."""
reader = SphinxStandaloneReader(app, parsers=app.registry.get_source_parsers())
source = SphinxFileInput(app, env, source=None, source_path=filename,
encoding=env.config.source_encoding)
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
source_class=SphinxDummySourceClass,
destination=NullOutput())
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source, filename)
pub.publish()
return pub.document
def _get_publisher(self, source_path):
extra_params = {
"initial_header_level": "2",
"syntax_highlight": "short",
"input_encoding": "utf-8",
}
user_params = self.settings.get("DOCUTILS_SETTINGS")
if user_params:
extra_params.update(user_params)
pub = Publisher(destination_class=StringOutput)
pub.set_components("standalone", "restructuredtext", "html")
pub.writer.translator_class = HTMLTranslator
pub.process_programmatic_settings(None, extra_params, None)
pub.set_source(source_path=source_path)
pub.publish()
return pub
def pypi_rest2html(source, output_encoding="unicode"):
"""
>>> pypi_rest2html("test!")
u'<p>test!</p>\n'
"""
settings_overrides = {
"raw_enabled": 0, # no raw HTML code
"file_insertion_enabled": 0, # no file/URL access
"halt_level": 2, # at warnings or errors, raise an exception
"report_level": 5, # never report problems with the reST code
}
# Convert reStructuredText to HTML using Docutils.
document = publish_doctree(source=source,
settings_overrides=settings_overrides)
for node in document.traverse():
if node.tagname == "#text":
continue
if node.hasattr("refuri"):
uri = node["refuri"]
elif node.hasattr("uri"):
uri = node["uri"]
else:
continue
o = urlparse(uri)
if o.scheme not in ALLOWED_SCHEMES:
raise TransformError("link scheme not allowed")
# now turn the transformed document into HTML
reader = readers.doctree.Reader(parser_name="null")
pub = Publisher(reader,
source=io.DocTreeInput(document),
destination_class=io.StringOutput)
pub.set_writer("html")
pub.process_programmatic_settings(None, settings_overrides, None)
pub.set_destination(None, None)
pub.publish()
parts = pub.writer.parts
output = parts["body"]
if output_encoding != "unicode":
output = output.encode(output_encoding)
return output
def render_map(builder, node):
if node:
doc = new_document(b('<partial node>'))
doc.append(node)
publisher = Publisher( source_class = DocTreeInput, destination_class=StringOutput)
publisher.set_components('standalone', 'restructuredtext', 'pseudoxml')
publisher.reader = DoctreeReader()
publisher.writer = DitaMapWriter(builder)
publisher.process_programmatic_settings(None, {'output_encoding': 'utf-8'}, None)
publisher.set_source(doc, None)
publisher.set_destination(None, None)
publisher.publish()
return publisher.writer.output
output = XML_HEAD
output += u"<map></map>"
return output
def _get_publisher(self, source_path):
extra_params = {
'initial_header_level': '2',
'syntax_highlight': 'short',
'input_encoding': 'utf-8'
}
user_params = self.settings.get('DOCUTILS_SETTINGS')
if user_params:
extra_params.update(user_params)
pub = Publisher(destination_class=StringOutput)
pub.set_components('standalone', 'restructuredtext', 'html')
pub.writer.translator_class = HTMLTranslator
pub.process_programmatic_settings(None, extra_params, None)
pub.set_source(source_path=source_path)
pub.publish()
return pub
def check_rst(self, rst):
pub = Publisher(reader=None, parser=None, writer=None, settings=None,
source_class=io.StringInput,
destination_class=io.StringOutput)
pub.set_components(reader_name='standalone',
parser_name='restructuredtext',
writer_name='pseudoxml')
pub.process_programmatic_settings(
settings_spec=None,
settings_overrides={'output_encoding': 'unicode'},
config_section=None,
)
pub.set_source(rst, source_path=None)
pub.set_destination(destination=None, destination_path=None)
output = pub.publish(enable_exit_status=False)
self.assertLess(pub.document.reporter.max_level, 0)
return output, pub
def check_rst(self, rst):
pub = Publisher(reader=None, parser=None, writer=None, settings=None,
source_class=io.StringInput,
destination_class=io.StringOutput)
pub.set_components(reader_name='standalone',
parser_name='restructuredtext',
writer_name='pseudoxml')
pub.process_programmatic_settings(
settings_spec=None,
settings_overrides={'output_encoding': 'unicode'},
config_section=None,
)
pub.set_source(rst, source_path=None)
pub.set_destination(destination=None, destination_path=None)
output = pub.publish(enable_exit_status=False)
self.assertLess(pub.document.reporter.max_level, 0)
return output, pub
def publish_doctree(source, logger):
# type: (str, Any) -> Node
pub = Publisher(reader=None,
parser=None,
writer=None,
settings=None,
source_class=io.StringInput,
destination_class=io.NullOutput)
pub.reader = Reader(None, 'restructuredtext')
pub.reader.doc_logger = logger
pub.set_writer('null')
pub.parser = pub.reader.parser
pub.process_programmatic_settings(None, None, None)
pub.set_source(source, None)
pub.set_destination(None, None)
output = pub.publish(enable_exit_status=False)
return pub.document
class HTMLGenerator: '''Really simple HTMLGenerator starting from publish_parts. It reuses the docutils.core.Publisher class, which means it is *not* threadsafe. ''' def __init__(self, settings_spec=None, settings_overrides=dict(report_level=5, halt_level=5), config_section='general'): self.pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=io.StringInput, destination_class=io.StringOutput) self.pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='html') # hack: JEP-0071 does not allow HTML char entities, so we hack our way # out of it. # — == u"\u2014" # a setting to only emit charater entities in the writer would be nice # FIXME: several are emitted, and they are explicitly forbidden # in the JEP # == u"\u00a0" self.pub.writer.translator_class.attribution_formats['dash'] = ( u'\u2014', '') self.pub.process_programmatic_settings(settings_spec, settings_overrides, config_section) def create_xhtml(self, text, destination=None, destination_path=None, enable_exit_status=None): ''' Create xhtml for a fragment of IM dialog. We can use the source_name to store info about the message.''' self.pub.set_source(text, None) self.pub.set_destination(destination, destination_path) output = self.pub.publish(enable_exit_status=enable_exit_status) # kludge until we can get docutils to stop generating (rare) # entities return u'\u00a0'.join(self.pub.writer.parts['fragment'].strip().split( ' '))
def read_doc(app, env, filename):
# type: (Sphinx, BuildEnvironment, unicode) -> nodes.document
"""Parse a document and convert to doctree."""
input_class = app.registry.get_source_input(filename)
reader = SphinxStandaloneReader(app)
source = input_class(app, env, source=None, source_path=filename,
encoding=env.config.source_encoding)
parser = app.registry.create_source_parser(app, filename)
pub = Publisher(reader=reader,
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxDummySourceClass,
destination=NullOutput())
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source, filename)
pub.publish()
return pub.document
def aoeuaoeuaoeu_docs(f):
'''
I couldn't figure out how to use the docutils docstring parser, so I wrote
my own. Can somebody show me the right way to do this?
'''
raise NotImplementedError
from docutils.core import Publisher
from docutils.io import StringInput
pub = Publisher(None, None, None, settings = settings,
source_class = StringInput,
destination_class = destination_class)
pub.set_components('standalone', 'restructuredtext', 'pseudoxml')
pub.process_programmatic_settings(
settings_spec, settings_overrides, config_section)
pub.set_source(f.__doc__, f.__name__)
pub.set_destination(None, f.__name__)
output = pub.publish(enable_exit_status = False)
return output, pub
return publish_parts(f.__doc__, source_class = StringInput, source_path = f.__name__)
def read_doc(app, env, filename):
# type: (Sphinx, BuildEnvironment, str) -> nodes.document
"""Parse a document and convert to doctree."""
# set up error_handler for the target document
error_handler = UnicodeDecodeErrorHandler(env.docname)
codecs.register_error('sphinx', error_handler) # type: ignore
reader = SphinxStandaloneReader(app)
filetype = get_filetype(app.config.source_suffix, filename)
parser = app.registry.create_source_parser(app, filetype)
if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == ():
# a workaround for recommonmark
# If recommonmark.AutoStrictify is enabled, the parser invokes reST parser
# internally. But recommonmark-0.4.0 does not provide settings_spec for reST
# parser. As a workaround, this copies settings_spec for RSTParser to the
# CommonMarkParser.
parser.settings_spec = RSTParser.settings_spec
input_class = app.registry.get_source_input(filetype)
if input_class:
# Sphinx-1.8 style
source = input_class(app, env, source=None, source_path=filename, # type: ignore
encoding=env.config.source_encoding)
pub = Publisher(reader=reader, # type: ignore
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxDummySourceClass,
destination=NullOutput())
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source, filename)
else:
# Sphinx-2.0 style
pub = Publisher(reader=reader,
parser=parser,
writer=SphinxDummyWriter(),
source_class=SphinxFileInput,
destination=NullOutput())
pub.process_programmatic_settings(None, env.settings, None)
pub.set_source(source_path=filename)
pub.publish()
return pub.document
def parts_from_doctree(document, destination_path=None,
writer=Writer(), writer_name='html4css2',
settings=None, settings_spec=None,
settings_overrides=None, config_section=None,
enable_exit_status=None):
"""
A version of :func:`docutils.core.publish_from_doctree` that supplies
the "parts" rather than a regular string representation.
"""
reader = Reader(parser_name='null')
pub = Publisher(reader, None, writer,
source=io.DocTreeInput(document),
destination_class=io.StringOutput, settings=settings)
if not writer and writer_name:
pub.set_writer(writer_name)
pub.process_programmatic_settings(
settings_spec, settings_overrides, config_section)
pub.set_destination(None, destination_path)
output = pub.publish()
return pub.writer.parts
def parts_from_doctree(document, destination_path=None,
writer=None, writer_name='pseudoxml',
settings=None, settings_spec=None,
settings_overrides=None, config_section=None,
enable_exit_status=None):
"""
Set up & run a `Publisher` to render from an existing document
tree data structure, for programmatic use with string I/O. Return
the encoded string output.
Note that document.settings is overridden; if you want to use the settings
of the original `document`, pass settings=document.settings.
Also, new document.transformer and document.reporter objects are
generated.
For encoded string output, be sure to set the 'output_encoding' setting to
the desired encoding. Set it to 'unicode' for unencoded Unicode string
output. Here's one way::
publish_from_doctree(
..., settings_overrides={'output_encoding': 'unicode'})
Parameters: `document` is a `docutils.nodes.document` object, an existing
document tree.
Other parameters: see `publish_programmatically`.
"""
reader = docutils.readers.doctree.Reader(parser_name='null')
pub = Publisher(reader, None, writer,
source=io.DocTreeInput(document),
destination_class=io.StringOutput, settings=settings)
if not writer and writer_name:
pub.set_writer(writer_name)
pub.process_programmatic_settings(
settings_spec, settings_overrides, config_section)
pub.set_destination(None, destination_path)
pub.publish(enable_exit_status=enable_exit_status)
return pub.writer.parts
def sandbox_partial_builder(doctree, env):
env.process_downloads(env.docname, doctree)
for domain in env.domains.values():
domain.process_doc(env, env.docname, doctree)
env.resolve_references(doctree, env.docname, env.app.builder)
if not hasattr(env.app.builder, 'dlpath'):
# FIXME: builders.html.StandaloneHTMLBuilder.write_doc で設定される属性のため、この時点では存在しない
env.app.builder.dlpath = '_downloads'
env.app.builder.fignumbers = env.toc_secnumbers.get(env.docname, {})
env.app.builder.secnumbers = env.toc_secnumbers.get(env.docname, {})
pub = Publisher(
reader=SandboxDoctreeReader(),
writer=Writer(env.app.builder),
source=DocTreeInput(doctree),
destination_class=StringOutput,
)
pub.set_components(None, 'restructuredtext', None)
defaults = env.settings.copy()
defaults['output_encoding'] = 'unicode'
pub.process_programmatic_settings(None, defaults, None)
pub.set_destination(None, None)
out = pub.publish(enable_exit_status=False)
return pub.writer.parts['fragment']
def __init__(self, app):
# After much digging through source code, this is how we emulate Sphinx's builtin reST parsing state
# https://github.com/sphinx-doc/sphinx/blob/68cc0f7e94f360a2c62ebcb761f8096e04ebf07f/sphinx/io.py#L204
# Here we're bypassing the RST Parser, and just doing the relevant code ops
parser = app.registry.create_source_parser(app, "restructuredtext")
# autosummary uses tab width 8; not set by publisher/env for some reason
settings = dict(app.env.settings)
if "tab_width" not in settings:
settings["tab_width"] = 8
p2 = Publisher()
p2.process_programmatic_settings(None, settings, None)
document = new_document('dummy_kissapi_source',
p2.settings) # (source path, settings)
document.reporter = NullReporter()
state_machine = RSTStateMachine(
state_classes, 'Body') # (state machine classes, initial state)
# needed to set various self.[attr] values of state_machine
state_machine.run([""], document) # (input lines, document)
# the directive attrs that are needed
self.state = state_machine.get_state()
self.env = app.env
self.lineno = 0
def read_initial_doctree(filename: pathlib.Path,
logger: logging.Logger) -> Optional[nodes.document]:
"""Parse the given reStructuredText file into its "initial" doctree.
An "initial" doctree can be thought of as the abstract syntax tree of a
reStructuredText document. This method disables all role and directives
from being executed, instead they are replaced with nodes that simply
represent that they exist.
Parameters
----------
filename
Returns the doctree that corresponds with the given file.
logger
Logger to log debug info to.
"""
parser = Parser()
if filename.suffix.replace(".", "") not in parser.supported:
return None
logger.debug("Getting initial doctree for: '%s'", filename)
with disable_roles_and_directives():
publisher = Publisher(
reader=InitialDoctreeReader(logger),
parser=parser,
writer=DummyWriter(),
source_class=FileInput,
destination=NullOutput(),
)
publisher.process_programmatic_settings(None, default_settings, None)
publisher.set_source(source_path=str(filename))
publisher.publish()
return publisher.document
def _get_publisher(self, source_path, translator_class=BlogHTMLTranslator):
extra_params = {
"initial_header_level": "2",
"syntax_highlight": "short",
"input_encoding": "utf-8",
"exit_status_level": 2,
"language_code": self._language_code,
"halt_level": 2,
"traceback": True,
"warning_stream": StringIO(),
"embed_stylesheet": False,
}
user_params = self.settings.get("DOCUTILS_SETTINGS")
if user_params:
extra_params.update(user_params)
pub = Publisher(writer=self.writer_class(),
destination_class=StringOutput)
pub.set_components("standalone", "restructuredtext", "html")
pub.writer.translator_class = translator_class
pub.process_programmatic_settings(None, extra_params, None)
pub.set_source(source_path=source_path)
pub.publish()
return pub
class _PydocParser:
def __init__(self):
# Set up the instance we'll be using to render docstrings.
self.errors = []
self.writer = _DocumentPseudoWriter()
self.publisher = Publisher(_EpydocReader(self.errors),
writer=self.writer,
source_class=io.StringInput)
self.publisher.set_components('standalone', 'restructuredtext',
'pseudoxml')
settings_overrides={
'report_level':10000,
'halt_level':10000,
'warning_stream':None,
}
self.publisher.process_programmatic_settings(None,
settings_overrides, None)
self.publisher.set_destination()
def parse_docstring(self, docstring, errors):
"""Parse a docstring for eventual transformation into HTML
This function is a replacement for parse_docstring from
epydoc.markup.restructuredtext.parse_docstring. This function reuses
the Publisher instance while the original did not. Using This
function yields significantly faster WADL generation for complex
systems.
"""
# Clear any errors from previous calls.
del self.errors[:]
self.publisher.set_source(docstring, None)
self.publisher.publish()
# Move any errors into the caller-provided list.
errors[:] = self.errors[:]
return ParsedRstDocstring(self.writer.document)
def write_doc(self, docname, doctree):
outfilename = path.join(self.outdir, self.file_transform(docname))
logger.debug(outfilename)
reader = docutils.readers.doctree.Reader(parser_name='null')
#Note, want to use our OdtWriter class here - but it doesn't work yet
writer = Writer()
pub = Publisher(reader,
None,
writer,
settings=None,
source=io.DocTreeInput(doctree),
destination_class=io.BinaryFileOutput)
settings_spec = None
settings_overrides = {'output_encoding': 'unicode'}
config_section = None
pub.process_programmatic_settings(settings_spec, settings_overrides,
config_section)
destination = None
destination_path = outfilename
pub.set_destination(destination, destination_path)
output = pub.publish(enable_exit_status=False)
def get_doctree(path, **kwargs):
"""
Obtain a Sphinx doctree from the RST file at ``path``.
Performs no Releases-specific processing; this code would, ideally, be in
Sphinx itself, but things there are pretty tightly coupled. So we wrote
this.
Any additional kwargs are passed unmodified into an internal `make_app`
call.
:param str path: A relative or absolute file path string.
:returns:
A two-tuple of the generated ``sphinx.application.Sphinx`` app and the
doctree (a ``docutils.document`` object).
.. versionchanged:: 1.6
Added support for passing kwargs to `make_app`.
"""
root, filename = os.path.split(path)
docname, _ = os.path.splitext(filename)
# TODO: this only works for top level changelog files (i.e. ones where
# their dirname is the project/doc root)
app = make_app(srcdir=root, **kwargs)
# Create & init a BuildEnvironment. Mm, tasty side effects.
app._init_env(freshenv=True)
env = app.env
# More arity/API changes: Sphinx 1.3/1.4-ish require one to pass in the app
# obj in BuildEnvironment.update(); modern Sphinx performs that inside
# Application._init_env() (which we just called above) and so that kwarg is
# removed from update(). EAFP.
kwargs = dict(
config=app.config,
srcdir=root,
doctreedir=app.doctreedir,
app=app,
)
try:
env.update(**kwargs)
except TypeError:
# Assume newer Sphinx w/o an app= kwarg
del kwargs['app']
env.update(**kwargs)
# Code taken from sphinx.environment.read_doc; easier to manually call
# it with a working Environment object, instead of doing more random crap
# to trick the higher up build system into thinking our single changelog
# document was "updated".
env.temp_data['docname'] = docname
env.app = app
# NOTE: SphinxStandaloneReader API changed in 1.4 :(
reader_kwargs = {
'app': app,
'parsers': env.config.source_parsers,
}
if sphinx.version_info[:2] < (1, 4):
del reader_kwargs['app']
# This monkeypatches (!!!) docutils to 'inject' all registered Sphinx
# domains' roles & so forth. Without this, rendering the doctree lacks
# almost all Sphinx magic, including things like :ref: and :doc:!
with sphinx_domains(env):
reader = SphinxStandaloneReader(**reader_kwargs)
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, env.settings, None)
# NOTE: docname derived higher up, from our given path
src_path = env.doc2path(docname)
source = SphinxFileInput(
app,
env,
source=None,
source_path=src_path,
encoding=env.config.source_encoding,
)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
return app, pub.document
def env_updated(app, env):
config = app.builder.config
doctree = env.get_doctree(config.master_doc)
from sphinx import addnodes
toctrees = []
for toctreenode in doctree.traverse(addnodes.toctree):
toctree = env.resolve_toctree(config.master_doc, app.builder,
toctreenode, prune = False, includehidden = True, maxdepth = 0,
collapse = False)
toctrees.append(toctree)
if not toctrees:
toc = None
else:
toc = toctrees[0]
for toctree in toctrees[1:]:
toc.extend(toctree.children)
# toc = env.get_toctree_for(config.master_doc, app.builder, False)
node = toc
doc = new_document(b('<partial node>'))
doc.append(node)
pub = Publisher(
source_class = DocTreeInput,
destination_class=StringOutput)
pub.set_components('standalone', 'restructuredtext', 'pseudoxml')
pub.reader = DoctreeReader()
pub.writer = Writer()
pub.writer.format = 'pseudoxml'
pub.process_programmatic_settings(
None, {'output_encoding': 'unicode'}, None)
pub.set_source(doc, None)
pub.set_destination(None, None)
pub.publish()
import xml.etree.cElementTree as ET
from cStringIO import StringIO
#out = re.sub(r'^<!DOCTYPE[^>]*>\s*', '<?xml version="1.0"?>', pub.writer.output)
out = pub.writer.output.encode('utf-8').replace(' encoding="unicode"', ' encoding="utf-8"')
#pprint.pprint(out)
doctree = ET.fromstring(out)
#pprint.pprint(doctree)
#pprint.pprint(dir(doctree))
if hasattr(doctree, 'getroot'):
doctree = doctree.getroot()
#root = doctree.getroot()
fuzzy_find_entries = []
docs = {}
def indexentries(entry, links, cap = ''):
if links:
fuzzy_find_entries.append(dict(
href = links[0][1],
name = entry,
path = "index/%s/%s" % (char,cap),
info = "INDEX",
detail = '',
))
for i, (ismain, link) in enumerate(links[1:], start=1):
doclink = link.split('#', 1)[0]
docname = docs.get(doclink, i)
fuzzy_find_entries.append(dict(
href = link,
name = "%s (%s)" % (entry, docname),
path = "index/%s/%s" % (char, cap),
info = "INDEX",
detail = '',
))
return ''
else:
return entry
if app.config.html_findanything_add_topics:
if hasattr(doctree, 'iter'):
references = doctree.iter('reference')
else:
references = doctree.getiterator('reference')
# fuzzy_find_entries.append(dict(
# href = link,
# name = entry,
# path = "index/%s/" % char,
# info = "INDEX",
# detail = '',
# ))
refset = set()
for ref in references:
refuri = ref.attrib['refuri']
docs[refuri] = ref.text
path = "/"
if "/" in refuri:
path = refuri.rsplit('/', 1)[0]+"/"
if '#' in refuri:
docname = docs.get(refuri.split('#', 1)[0])
if docname:
path += docname + "/"
info = 'SECTION'
else:
info = 'PAGE'
e = dict(
href = ref.attrib['refuri'],
name = ref.text,
info = info,
path = path,
detail = '',
)
refid = "%(href)s^%(path)s^%(name)s^%(info)s" % e
if refid in refset: continue
refset.add(refid)
fuzzy_find_entries.append(e)
if app.config.html_findanything_add_indexentries:
genindex = env.create_index(app.builder)
for char,char_list in genindex:
for entry, (links, subitems) in char_list:
cap = indexentries(entry, links)
if subitems:
if cap: cap += '/'
for subname, sublinks in subitems:
indexentries(subname, sublinks, cap=cap)
s = json.dumps(fuzzy_find_entries)
static_dir = os.path.join(app.builder.outdir, '_static')
if not os.path.exists(static_dir):
os.makedirs(static_dir)
with open(os.path.join(static_dir, 'fuzzyindex.js'), 'wb') as f:
f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_ENTRIES = %s;" % s);
if app.config.html_findanything_use_cached_hits:
f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_USE_CACHED_HITS = true;")
f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_WIDTH = '%s';"
% app.config.html_findanything_width);
def create_toc(doctree, depth=9223372036854775807, writer_name='html',
exclude_first_section=True, href_prefix=None, id_prefix='toc-ref-'):
"""
Create a Table of Contents (TOC) from the given doctree
Returns: (docutils.core.Publisher instance, output string)
`writer_name`: represents a reST writer name and determines the type of
output returned.
Example:
pub = blazeutils.rst.rst2pub(toc_rst)
pub, html_output = blazeutils.rst.create_toc(pub.document)
# a full HTML document (probably not what you want most of the time)
print html_output
# just the TOC
print pub.writer.parts['body']
"""
# copy the doctree since Content alters some settings on the original
# document
doctree = doctree.deepcopy()
# we want to be able to customize ids to avoid clashes if needed
doctree.settings.auto_id_prefix = id_prefix
details = {
'depth': depth,
}
# Assuming the document has one primary heading and then sub-sections, we
# want to be able to give just the sub-sections
startnode = None
if exclude_first_section:
nodes = doctree.traverse(docutils.nodes.section)
if nodes:
startnode = nodes[0]
# use the Contents transform to build the TOC node structure from the
# document
c = Contents(doctree)
# this startnode isn't really used as the start node, its only used for
# to pull settings from
c.startnode = BlankObject(details=details)
# since this toc is detached from the rest of the document, we don't want
# backlinks
c.backlinks = 'none'
# build the nodes
toc_nodes = c.build_contents(startnode or doctree)
# create a new document with the new nodes
toc_doc = new_document(None)
toc_doc += toc_nodes
# fix fragements that reference the same page
if href_prefix:
prefix_refids(toc_doc, href_prefix)
# setup a publisher and publish from the TOC document
reader = docutils.readers.doctree.Reader(parser_name='null')
pub = Publisher(
reader, None, None,
source=io.DocTreeInput(toc_doc),
destination_class=io.StringOutput
)
pub.set_writer(writer_name)
final_settings_overrides = default_rst_opts.copy()
pub.process_programmatic_settings(
None, final_settings_overrides, None)
output = pub.publish()
return pub, output
def read_doc(self, docname, app=None):
# type: (unicode, Sphinx) -> None
"""Parse a file and add/update inventory entries for the doctree."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
docutilsconf = path.join(self.srcdir, 'docutils.conf')
# read docutils.conf from source dir, not from current dir
OptionParser.standard_config_files[1] = docutilsconf
if path.isfile(docutilsconf):
self.note_dependency(docutilsconf)
with sphinx_domains(self):
if self.config.default_role:
role_fn, messages = roles.role(self.config.default_role, english,
0, dummy_reporter)
if role_fn:
roles._roles[''] = role_fn
else:
logger.warning('default role %s not found', self.config.default_role,
location=docname)
codecs.register_error('sphinx', self.warn_and_replace) # type: ignore
# publish manually
reader = SphinxStandaloneReader(self.app, parsers=self.config.source_parsers)
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, self.settings, None)
src_path = self.doc2path(docname)
source = SphinxFileInput(app, self, source=None, source_path=src_path,
encoding=self.config.source_encoding)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
doctree = pub.document
# post-processing
for domain in itervalues(self.domains):
domain.process_doc(self, docname, doctree)
# allow extension-specific post-processing
if app:
app.emit('doctree-read', doctree)
# store time of reading, for outdated files detection
# (Some filesystems have coarse timestamp resolution;
# therefore time.time() can be older than filesystem's timestamp.
# For example, FAT32 has 2sec timestamp resolution.)
self.all_docs[docname] = max(
time.time(), path.getmtime(self.doc2path(docname)))
if self.versioning_condition:
old_doctree = None
if self.versioning_compare:
# get old doctree
try:
with open(self.doc2path(docname,
self.doctreedir, '.doctree'), 'rb') as f:
old_doctree = pickle.load(f)
except EnvironmentError:
pass
# add uids for versioning
if not self.versioning_compare or old_doctree is None:
list(add_uids(doctree, self.versioning_condition))
else:
list(merge_doctrees(
old_doctree, doctree, self.versioning_condition))
# make it picklable
doctree.reporter = None
doctree.transformer = None
doctree.settings.warning_stream = None
doctree.settings.env = None
doctree.settings.record_dependencies = None
# cleanup
self.temp_data.clear()
self.ref_context.clear()
roles._roles.pop('', None) # if a document has set a local default role
# save the parsed doctree
doctree_filename = self.doc2path(docname, self.doctreedir,
'.doctree')
ensuredir(path.dirname(doctree_filename))
with open(doctree_filename, 'wb') as f:
pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
def create_toc(doctree, depth=9223372036854775807, writer_name='html',
exclude_first_section=True, href_prefix=None, id_prefix='toc-ref-'):
"""
Create a Table of Contents (TOC) from the given doctree
Returns: (docutils.core.Publisher instance, output string)
`writer_name`: represents a reST writer name and determines the type of
output returned.
Example:
pub = blazeutils.rst.rst2pub(toc_rst)
pub, html_output = blazeutils.rst.create_toc(pub.document)
# a full HTML document (probably not what you want most of the time)
print html_output
# just the TOC
print pub.writer.parts['body']
"""
# copy the doctree since Content alters some settings on the original
# document
doctree = doctree.deepcopy()
# we want to be able to customize ids to avoid clashes if needed
doctree.settings.auto_id_prefix = id_prefix
details = {
'depth': depth,
}
# Assuming the document has one primary heading and then sub-sections, we
# want to be able to give just the sub-sections
startnode = None
if exclude_first_section:
nodes = doctree.traverse(docutils.nodes.section)
if nodes:
startnode = nodes[0]
# use the Contents transform to build the TOC node structure from the
# document
c = Contents(doctree)
# this startnode isn't really used as the start node, its only used for
# to pull settings from
c.startnode = BlankObject(details=details)
# since this toc is detached from the rest of the document, we don't want
# backlinks
c.backlinks = 'none'
# build the nodes
toc_nodes = c.build_contents(startnode or doctree)
# create a new document with the new nodes
toc_doc = new_document(None)
toc_doc += toc_nodes
# fix fragements that reference the same page
if href_prefix:
prefix_refids(toc_doc, href_prefix)
# setup a publisher and publish from the TOC document
reader = docutils.readers.doctree.Reader(parser_name='null')
pub = Publisher(
reader, None, None,
source=io.DocTreeInput(toc_doc),
destination_class=io.StringOutput
)
pub.set_writer(writer_name)
final_settings_overrides = default_rst_opts.copy()
pub.process_programmatic_settings(
None, final_settings_overrides, None)
output = pub.publish()
return pub, output
from docutils.core import publish_cmdline, default_description,\
publish_doctree,\
Publisher
from dotmpe.du.ext.writer.formresults import Writer as FormResultsWriter
#publish_cmdline(reader=FormReader(), writer_name='pseudoxml')#writer=xhtmlform.Writer())
source_path = sys.argv[1]
source = open(source_path)
#doctree = publish_doctree(source, reader=FormReader())#, writer_name='pseudoxml')
#print doctree
pub = Publisher(FormReader(),
writer=FormResultsWriter(),
source_class=io.FileInput,
destination_class=io.StringOutput)#, parser, writer, settings=settings,
# source_class=source_class,
# destination_class=destination_class)
pub.set_components(None, 'rst', 'pseudoxml')
pub.process_programmatic_settings(None, None, None)
# settings_spec, settings_overrides, config_section)
pub.set_source(source, source_path)
pub.set_destination(None, None)#destination, destination_path)
output = pub.publish(enable_exit_status=False)
#print pub.document.form_processor.fields
#print pub.document.form_processor.messages
#print pub.document.form_processor.values
print output
class RstPublisher:
##############################################
def __init__(self):
self._publisher = Publisher(source_class=StringInput,
destination_class=StringOutput)
self._publisher.set_components('standalone', 'restructuredtext',
'html')
self._publisher.writer.translator_class = MyHTMLTranslator
self._publisher.process_programmatic_settings(None, extra_params, None)
##############################################
def init_plugins(self, plugins):
site.addsitedir(str(PurePath(__file__).parent / 'RstPlugins'))
for plugin in plugins:
print('Register {}'.format(plugin))
plugin = __import__(plugin, globals(), locals(), str('module'))
plugin.register(self)
##############################################
def register_directive(self, directive_name, directive_class):
directives.register_directive(directive_name, directive_class)
##############################################
def register_role(self, role_name, role_class):
roles.register_local_role(role_name, role_class)
##############################################
def register_node(self, node_class, visit, depart):
print(node_class.__name__, visit, depart)
setattr(MyHTMLTranslator, 'visit_' + node_class.__name__, visit)
setattr(MyHTMLTranslator, 'depart_' + node_class.__name__, depart)
##############################################
def publish(self, source, header_level=None, report_level=None):
settings_overrides = DOCUTILS_RENDERER_SETTINGS.copy()
if header_level is not None: # starts from 1
settings_overrides['initial_header_level'] = header_level
if report_level is not None: # starts from 1 too
settings_overrides['report_level'] = 0 # report_level
self._publisher.set_source(source=source)
self._publisher.publish(enable_exit_status=True)
parts = self._publisher.writer.parts
return parts[
'html_body'] # parts['body_pre_docinfo'] + parts['fragment']
def process_targets(site, logger, source, permalink):
"""Process the target locations in the reST files."""
site.processing_targets = True
reader = Reader()
reader.l_settings = {"source": source}
with open(source, "r", encoding="utf8") as in_file:
data = in_file.read()
pub = Publisher(
reader=reader,
parser=None,
writer=None,
settings=None,
source_class=StringInput,
destination_class=StringOutput,
)
pub.set_components(None, "restructuredtext", "html")
# Reading the file will generate output/errors that we don't care about
# at this stage. The report_level = 5 means no output
pub.process_programmatic_settings(settings_spec=None,
settings_overrides={"report_level": 5},
config_section=None)
pub.set_source(data, None)
pub.set_destination(None, None)
pub.publish()
document = pub.document
site.processing_targets = False
# Code based on Sphinx std domain
for name, is_explicit in document.nametypes.items():
if not is_explicit:
continue
labelid = document.nameids[name]
if labelid is None:
continue
node = document.ids[labelid]
if node.tagname == "target" and "refid" in node:
node = document.ids.get(node["refid"])
labelid = node["names"][0]
if (node.tagname == "footnote" or "refuri" in node
or node.tagname.startswith("desc_")):
continue
if name in site.ref_targets:
logger.warn(
"Duplicate label {dup}, other instance in {other}".format(
dup=name, other=site.ref_targets[name][0]))
site.anon_ref_targets[name] = permalink, labelid
def clean_astext(node):
"""Like node.astext(), but ignore images.
Taken from sphinx.util.nodes
"""
node = node.deepcopy()
for img in node.traverse(nodes.image):
img["alt"] = ""
for raw in node.traverse(nodes.raw):
raw.parent.remove(raw)
return node.astext()
if node.tagname in ("section", "rubric"):
sectname = clean_astext(node[0])
else:
continue
site.ref_targets[name] = permalink, labelid, sectname
