def render(self): key = 'cia.apps.blog.%d' % self.id parts = cache.get(key) if not parts: # Convert the reST markup to a document tree document = publish_doctree(source=self.content) visitor = ImageTranslator(document) document.walkabout(visitor) # # Publish that document tree as HTML. We can't use any of # the simpler methods in docutils.core, since we need # access to writer.parts # reader = doctree.Reader(parser_name='null') pub = Publisher(reader, None, None, source=DocTreeInput(document), destination_class=StringOutput) pub.set_writer('html4css1') pub.process_programmatic_settings(None, { 'cloak_email_addresses': True, 'initial_header_level': 2, }, None) pub.publish() parts = pub.writer.parts cache.set(key, parts) return parts
def get_sphinx(): sphinx = getattr(local_data, 'sphinx', None) if sphinx is None: sphinx = Sphinx(tempdir, tempdir, tempdir, tempdir, 'json', status=None, warning=None) sphinx.builder.translator_class = CustomHTMLTranslator sphinx.env.patch_lookup_functions() sphinx.env.temp_data['docname'] = 'text' sphinx.env.temp_data['default_domain'] = 'py' pub = Publisher(reader=None, parser=None, writer=HTMLWriter(sphinx.builder), source_class=io.StringInput, destination_class=io.NullOutput) pub.set_components('standalone', 'restructuredtext', None) pub.process_programmatic_settings(None, sphinx.env.settings, None) pub.set_destination(None, None) sphinx.publisher = pub local_data.sphinx = sphinx return sphinx, sphinx.publisher
def parse_docstring(doc): p = Publisher(source=doc, source_class=io.StringInput) p.set_reader('standalone', p.parser, 'restructuredtext') p.writer = Writer() p.process_programmatic_settings(None, None, None) p.set_source(doc, None) return p.publish()
def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document """Parse a document and convert to doctree.""" filetype = get_filetype(app.config.source_suffix, filename) input_class = app.registry.get_source_input(filetype) reader = SphinxStandaloneReader(app) source = input_class(app, env, source=None, source_path=filename, encoding=env.config.source_encoding) parser = app.registry.create_source_parser(app, filetype) if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == (): # a workaround for recommonmark # If recommonmark.AutoStrictify is enabled, the parser invokes reST parser # internally. But recommonmark-0.4.0 does not provide settings_spec for reST # parser. As a workaround, this copies settings_spec for RSTParser to the # CommonMarkParser. parser.settings_spec = RSTParser.settings_spec pub = Publisher(reader=reader, parser=parser, writer=SphinxDummyWriter(), source_class=SphinxDummySourceClass, destination=NullOutput()) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source, filename) pub.publish() return pub.document
def read_doc(app: "Sphinx", env: BuildEnvironment, filename: str) -> nodes.document: """Parse a document and convert to doctree.""" # set up error_handler for the target document error_handler = UnicodeDecodeErrorHandler(env.docname) codecs.register_error('sphinx', error_handler) # type: ignore reader = SphinxStandaloneReader() reader.setup(app) filetype = get_filetype(app.config.source_suffix, filename) parser = app.registry.create_source_parser(app, filetype) if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == ( ): # a workaround for recommonmark # If recommonmark.AutoStrictify is enabled, the parser invokes reST parser # internally. But recommonmark-0.4.0 does not provide settings_spec for reST # parser. As a workaround, this copies settings_spec for RSTParser to the # CommonMarkParser. parser.settings_spec = RSTParser.settings_spec pub = Publisher(reader=reader, parser=parser, writer=SphinxDummyWriter(), source_class=SphinxFileInput, destination=NullOutput()) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source_path=filename) pub.publish() return pub.document
def render(self): key = 'cia.apps.blog.%d' % self.id parts = cache.get(key) if not parts: # Convert the reST markup to a document tree document = publish_doctree(source = self.content) visitor = ImageTranslator(document) document.walkabout(visitor) # # Publish that document tree as HTML. We can't use any of # the simpler methods in docutils.core, since we need # access to writer.parts # reader = doctree.Reader(parser_name='null') pub = Publisher(reader, None, None, source = DocTreeInput(document), destination_class = StringOutput) pub.set_writer('html4css1') pub.process_programmatic_settings(None, { 'cloak_email_addresses': True, 'initial_header_level': 2, }, None) pub.publish() parts = pub.writer.parts cache.set(key, parts) return parts
def pypi_render(source): """ Copied (and slightly adapted) from pypi.description_tools """ ALLOWED_SCHEMES = '''file ftp gopher hdl http https imap mailto mms news nntp prospero rsync rtsp rtspu sftp shttp sip sips snews svn svn+ssh telnet wais irc'''.split() settings_overrides = { "raw_enabled": 0, # no raw HTML code "file_insertion_enabled": 0, # no file/URL access "halt_level": 2, # at warnings or errors, raise an exception "report_level": 5, # never report problems with the reST code } # capture publishing errors, they go to stderr old_stderr = sys.stderr sys.stderr = s = StringIO.StringIO() parts = None try: # Convert reStructuredText to HTML using Docutils. document = publish_doctree(source=source, settings_overrides=settings_overrides) for node in document.traverse(): if node.tagname == '#text': continue if node.hasattr('refuri'): uri = node['refuri'] elif node.hasattr('uri'): uri = node['uri'] else: continue o = urlparse.urlparse(uri) if o.scheme not in ALLOWED_SCHEMES: raise TransformError('link scheme not allowed') # now turn the transformed document into HTML reader = readers.doctree.Reader(parser_name='null') pub = Publisher(reader, source=io.DocTreeInput(document), destination_class=io.StringOutput) pub.set_writer('html') pub.process_programmatic_settings(None, settings_overrides, None) pub.set_destination(None, None) pub.publish() parts = pub.writer.parts except: pass sys.stderr = old_stderr # original text if publishing errors occur if parts is None or len(s.getvalue()) > 0: return None else: return parts['body']
def process_description(source, output_encoding='unicode'): """Given an source string, returns an HTML fragment as a string. The return value is the contents of the <body> tag. Parameters: - `source`: A multi-line text string; required. - `output_encoding`: The desired encoding of the output. If a Unicode string is desired, use the default value of "unicode" . """ # Dedent all lines of `source`. source = trim_docstring(source) settings_overrides = { 'raw_enabled': 0, # no raw HTML code 'file_insertion_enabled': 0, # no file/URL access 'halt_level': 2, # at warnings or errors, raise an exception 'report_level': 5, # never report problems with the reST code } parts = None # Convert reStructuredText to HTML using Docutils. document = publish_doctree(source=source, settings_overrides=settings_overrides) for node in document.traverse(): if node.tagname == '#text': continue if node.hasattr('refuri'): uri = node['refuri'] elif node.hasattr('uri'): uri = node['uri'] else: continue o = urlparse(uri) if o.scheme not in ALLOWED_SCHEMES: raise TransformError('link scheme not allowed: {0}'.format(uri)) # now turn the transformed document into HTML reader = readers.doctree.Reader(parser_name='null') pub = Publisher(reader, source=io.DocTreeInput(document), destination_class=io.StringOutput) pub.set_writer('html') pub.process_programmatic_settings(None, settings_overrides, None) pub.set_destination(None, None) pub.publish() parts = pub.writer.parts output = parts['body'] if output_encoding != 'unicode': output = output.encode(output_encoding) return output
def inspect(filename, source_path=None): "returns the document object before any transforms)" from docutils.core import Publisher pub = Publisher(source_class=io.FileInput) pub.set_reader("standalone", None, "restructuredtext") pub.process_programmatic_settings(None, None, None) pub.set_source(source_path=source_path) pub.set_io() return pub.reader.read(pub.source, pub.parser, pub.settings)
def __init__(self): # publisher is used for html generation pub = Publisher(None, None, None, settings=None, source_class=StringInput, destination_class=StringOutput) pub.set_components('standalone', 'restructuredtext', 'html') pub.process_programmatic_settings(None, None, None) pub.set_destination(None, None) self.pub = pub
def get_html(self, body_only=True, content_only=False, noclasses=False): import sys import pygments_rest from docutils.core import Publisher from docutils.io import StringInput, StringOutput from cStringIO import StringIO settings = { 'doctitle_xform': 1, 'pep_references': 1, 'rfc_references': 1, 'footnote_references': 'superscript', 'output_encoding': 'unicode', 'report_level': 2, # 2=show warnings, 3=show only errors, 5=off (docutils.utils } if content_only: post_rst = self.get_rst(noclasses=noclasses) else: post_rst = render_to('post_single.rst', post=self, noclasses=noclasses) pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=StringInput, destination_class=StringOutput) pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='html') pub.process_programmatic_settings(settings_spec=None, settings_overrides=settings, config_section=None) pub.set_source(post_rst, source_path=self.module_path) pub.set_destination(None, None) errors_io = StringIO() real_stderr = sys.stderr sys.stderr = errors_io try: html_full = pub.publish(enable_exit_status=False) html_body = ''.join(pub.writer.html_body) finally: sys.stderr = real_stderr errors = errors_io.getvalue() self._process_rest_errors(errors) errors_io.close() return html_body if body_only else html_full
def inspect( filename, source_path=None, ): "returns the document object before any transforms)" from docutils.core import Publisher pub = Publisher(source_class=io.FileInput, ) pub.set_reader('standalone', None, "restructuredtext") pub.process_programmatic_settings(None, None, None) pub.set_source(source_path=source_path) pub.set_io() return pub.reader.read(pub.source, pub.parser, pub.settings)
def convert(infilename, outfilename): print "converting %s to %s" % (infilename, outfilename) pub = Publisher() pub.set_components('standalone', # reader 'restructuredtext', # parser 'latex') # writer (arg, will be discarded) pub.reader = StandaloneReader() pub.writer = VerseLaTeXWriter() pub.process_programmatic_settings(None, None, None) pub.set_source(source_path=infilename) pub.set_destination(destination_path=outfilename) pub.publish()
def get_html(self, body_only=True, content_only=False, noclasses=False): import sys import pygments_rest from docutils.core import Publisher from docutils.io import StringInput, StringOutput from cStringIO import StringIO settings = {'doctitle_xform' : 1, 'pep_references' : 1, 'rfc_references' : 1, 'footnote_references': 'superscript', 'output_encoding' : 'unicode', 'report_level' : 2, # 2=show warnings, 3=show only errors, 5=off (docutils.utils } if content_only: post_rst = self.get_rst(noclasses=noclasses) else: post_rst = render_to('post_single.rst', post=self, noclasses=noclasses) pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=StringInput, destination_class=StringOutput) pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='html') pub.process_programmatic_settings(settings_spec=None, settings_overrides=settings, config_section=None) pub.set_source(post_rst,source_path=self.module_path) pub.set_destination(None, None) errors_io = StringIO() real_stderr = sys.stderr sys.stderr = errors_io try: html_full = pub.publish(enable_exit_status=False) html_body = ''.join(pub.writer.html_body) finally: sys.stderr = real_stderr errors = errors_io.getvalue() self._process_rest_errors(errors) errors_io.close() return html_body if body_only else html_full
def convert(infilename, outfilename): print "converting %s to %s" % (infilename, outfilename) pub = Publisher() pub.set_components( 'standalone', # reader 'restructuredtext', # parser 'latex') # writer (arg, will be discarded) pub.reader = OptikReader() pub.writer = PyLaTeXWriter() pub.process_programmatic_settings(None, None, None) pub.set_source(source_path=infilename) pub.set_destination(destination_path=outfilename) pub.publish()
def process_labels(site, logger, source, post): site.processing_labels = True pub = Publisher(reader=Reader(), parser=None, writer=None) pub.set_components(None, 'restructuredtext', 'html') # Reading the file will generate output/errors that we don't care about # at this stage. The report_level = 5 means no output pub.process_programmatic_settings( settings_spec=None, settings_overrides={'report_level': 5}, config_section=None, ) pub.set_source(None, source) pub.publish() document = pub.document site.processing_labels = False # Code based on Sphinx std domain for name, is_explicit in document.nametypes.items(): if not is_explicit: continue labelid = document.nameids[name] if labelid is None: continue node = document.ids[labelid] if node.tagname == 'target' and 'refid' in node: node = document.ids.get(node['refid']) labelid = node['names'][0] if node.tagname == 'footnote' or 'refuri' in node or node.tagname.startswith( 'desc_'): continue if name in site.ref_labels: logger.warn( 'Duplicate label {dup}, other instance in {other}'.format( dup=name, other=site.ref_labels[name][0])) site.anon_ref_labels[name] = post.permalink(), labelid def clean_astext(node): """Like node.astext(), but ignore images. Taken from sphinx.util.nodes""" node = node.deepcopy() for img in node.traverse(nodes.image): img['alt'] = '' for raw in node.traverse(nodes.raw): raw.parent.remove(raw) return node.astext() if node.tagname in ('section', 'rubric'): sectname = clean_astext(node[0]) else: continue site.ref_labels[name] = post.permalink(), labelid, sectname
class HTMLGenerator: """ Really simple HTMLGenerator starting from publish_parts It reuses the docutils.core.Publisher class, which means it is *not* threadsafe. """ def __init__(self, settings_spec=None, settings_overrides=None, config_section='general'): if settings_overrides is None: settings_overrides = {'report_level': 5, 'halt_level': 5} self.pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=io.StringInput, destination_class=io.StringOutput) self.pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='html') # hack: JEP-0071 does not allow HTML char entities, so we hack our way # out of it. # — == u"\u2014" # a setting to only emit charater entities in the writer would be nice # FIXME: several are emitted, and they are explicitly forbidden # in the JEP # == u"\u00a0" self.pub.writer.translator_class.attribution_formats['dash'] = ( '\u2014', '') self.pub.process_programmatic_settings(settings_spec, settings_overrides, config_section) def create_xhtml(self, text, destination=None, destination_path=None, enable_exit_status=None): """ Create xhtml for a fragment of IM dialog. We can use the source_name to store info about the message """ self.pub.set_source(text, None) self.pub.set_destination(destination, destination_path) output = self.pub.publish(enable_exit_status=enable_exit_status) # kludge until we can get docutils to stop generating (rare) # entities return '\u00a0'.join( self.pub.writer.parts['fragment'].strip().split(' '))
def render_readme_like_pypi(source, output_encoding='unicode'): """ Render a ReST document just like PyPI does. """ # Dedent all lines of `source`. source = trim_docstring(source) settings_overrides = { 'raw_enabled': 0, # no raw HTML code 'file_insertion_enabled': 0, # no file/URL access 'halt_level': 2, # at warnings or errors, raise an exception 'report_level': 5, # never report problems with the reST code } parts = None # Convert reStructuredText to HTML using Docutils. document = publish_doctree(source=source, settings_overrides=settings_overrides) for node in document.traverse(): if node.tagname == '#text': continue if node.hasattr('refuri'): uri = node['refuri'] elif node.hasattr('uri'): uri = node['uri'] else: continue o = urlparse(uri) if o.scheme not in ALLOWED_SCHEMES: raise TransformError('link scheme not allowed: {0}'.format(uri)) # now turn the transformed document into HTML reader = readers.doctree.Reader(parser_name='null') pub = Publisher(reader, source=io.DocTreeInput(document), destination_class=io.StringOutput) pub.set_writer('html') pub.process_programmatic_settings(None, settings_overrides, None) pub.set_destination(None, None) pub.publish() parts = pub.writer.parts output = parts['body'] if output_encoding != 'unicode': output = output.encode(output_encoding) return output
def _get_publisher(self, source_path): extra_params = {'initial_header_level': '2', 'syntax_highlight': 'short', 'input_encoding': 'utf-8'} user_params = self.settings.get('DOCUTILS_SETTINGS') if user_params: extra_params.update(user_params) pub = Publisher(destination_class=StringOutput) pub.set_components('standalone', 'restructuredtext', 'html') pub.writer.translator_class = HTMLTranslator pub.process_programmatic_settings(None, extra_params, None) pub.set_source(source_path=source_path) pub.publish() return pub
def get_doctree(path): """ Obtain a Sphinx doctree from the RST file at ``path``. Performs no Releases-specific processing; this code would, ideally, be in Sphinx itself, but things there are pretty tightly coupled. So we wrote this. :param str path: A relative or absolute file path string. :returns: A two-tuple of the generated ``sphinx.application.Sphinx`` app and the doctree (a ``docutils.document`` object). """ root, filename = os.path.split(path) docname, _ = os.path.splitext(filename) # TODO: this only works for top level changelog files (i.e. ones where # their dirname is the project/doc root) app = make_app(srcdir=root) # Create & init a BuildEnvironment. Mm, tasty side effects. app._init_env(freshenv=True) env = app.env env.update(config=app.config, srcdir=root, doctreedir=app.doctreedir, app=app) # Code taken from sphinx.environment.read_doc; easier to manually call # it with a working Environment object, instead of doing more random crap # to trick the higher up build system into thinking our single changelog # document was "updated". env.temp_data['docname'] = docname env.app = app # NOTE: SphinxStandaloneReader API changed in 1.4 :( reader_kwargs = {'app': env.app, 'parsers': env.config.source_parsers} if sphinx.version_info[:2] < (1, 4): del reader_kwargs['app'] reader = SphinxStandaloneReader(**reader_kwargs) pub = Publisher(reader=reader, writer=SphinxDummyWriter(), destination_class=NullOutput) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, env.settings, None) # NOTE: docname derived higher up, from our given path src_path = env.doc2path(docname) source = SphinxFileInput(app, env, source=None, source_path=src_path, encoding=env.config.source_encoding) pub.source = source pub.settings._source = src_path pub.set_destination(None, None) pub.publish() return app, pub.document
def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document """Parse a document and convert to doctree.""" reader = SphinxStandaloneReader(app, parsers=app.registry.get_source_parsers()) source = SphinxFileInput(app, env, source=None, source_path=filename, encoding=env.config.source_encoding) pub = Publisher(reader=reader, writer=SphinxDummyWriter(), source_class=SphinxDummySourceClass, destination=NullOutput()) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source, filename) pub.publish() return pub.document
def _get_publisher(self, source_path): extra_params = { "initial_header_level": "2", "syntax_highlight": "short", "input_encoding": "utf-8", } user_params = self.settings.get("DOCUTILS_SETTINGS") if user_params: extra_params.update(user_params) pub = Publisher(destination_class=StringOutput) pub.set_components("standalone", "restructuredtext", "html") pub.writer.translator_class = HTMLTranslator pub.process_programmatic_settings(None, extra_params, None) pub.set_source(source_path=source_path) pub.publish() return pub
def pypi_rest2html(source, output_encoding="unicode"): """ >>> pypi_rest2html("test!") u'<p>test!</p>\n' """ settings_overrides = { "raw_enabled": 0, # no raw HTML code "file_insertion_enabled": 0, # no file/URL access "halt_level": 2, # at warnings or errors, raise an exception "report_level": 5, # never report problems with the reST code } # Convert reStructuredText to HTML using Docutils. document = publish_doctree(source=source, settings_overrides=settings_overrides) for node in document.traverse(): if node.tagname == "#text": continue if node.hasattr("refuri"): uri = node["refuri"] elif node.hasattr("uri"): uri = node["uri"] else: continue o = urlparse(uri) if o.scheme not in ALLOWED_SCHEMES: raise TransformError("link scheme not allowed") # now turn the transformed document into HTML reader = readers.doctree.Reader(parser_name="null") pub = Publisher(reader, source=io.DocTreeInput(document), destination_class=io.StringOutput) pub.set_writer("html") pub.process_programmatic_settings(None, settings_overrides, None) pub.set_destination(None, None) pub.publish() parts = pub.writer.parts output = parts["body"] if output_encoding != "unicode": output = output.encode(output_encoding) return output
def render_map(builder, node): if node: doc = new_document(b('<partial node>')) doc.append(node) publisher = Publisher( source_class = DocTreeInput, destination_class=StringOutput) publisher.set_components('standalone', 'restructuredtext', 'pseudoxml') publisher.reader = DoctreeReader() publisher.writer = DitaMapWriter(builder) publisher.process_programmatic_settings(None, {'output_encoding': 'utf-8'}, None) publisher.set_source(doc, None) publisher.set_destination(None, None) publisher.publish() return publisher.writer.output output = XML_HEAD output += u"<map></map>" return output
def _get_publisher(self, source_path): extra_params = { 'initial_header_level': '2', 'syntax_highlight': 'short', 'input_encoding': 'utf-8' } user_params = self.settings.get('DOCUTILS_SETTINGS') if user_params: extra_params.update(user_params) pub = Publisher(destination_class=StringOutput) pub.set_components('standalone', 'restructuredtext', 'html') pub.writer.translator_class = HTMLTranslator pub.process_programmatic_settings(None, extra_params, None) pub.set_source(source_path=source_path) pub.publish() return pub
def check_rst(self, rst): pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=io.StringInput, destination_class=io.StringOutput) pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='pseudoxml') pub.process_programmatic_settings( settings_spec=None, settings_overrides={'output_encoding': 'unicode'}, config_section=None, ) pub.set_source(rst, source_path=None) pub.set_destination(destination=None, destination_path=None) output = pub.publish(enable_exit_status=False) self.assertLess(pub.document.reporter.max_level, 0) return output, pub
def publish_doctree(source, logger): # type: (str, Any) -> Node pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=io.StringInput, destination_class=io.NullOutput) pub.reader = Reader(None, 'restructuredtext') pub.reader.doc_logger = logger pub.set_writer('null') pub.parser = pub.reader.parser pub.process_programmatic_settings(None, None, None) pub.set_source(source, None) pub.set_destination(None, None) output = pub.publish(enable_exit_status=False) return pub.document
class HTMLGenerator: '''Really simple HTMLGenerator starting from publish_parts. It reuses the docutils.core.Publisher class, which means it is *not* threadsafe. ''' def __init__(self, settings_spec=None, settings_overrides=dict(report_level=5, halt_level=5), config_section='general'): self.pub = Publisher(reader=None, parser=None, writer=None, settings=None, source_class=io.StringInput, destination_class=io.StringOutput) self.pub.set_components(reader_name='standalone', parser_name='restructuredtext', writer_name='html') # hack: JEP-0071 does not allow HTML char entities, so we hack our way # out of it. # — == u"\u2014" # a setting to only emit charater entities in the writer would be nice # FIXME: several are emitted, and they are explicitly forbidden # in the JEP # == u"\u00a0" self.pub.writer.translator_class.attribution_formats['dash'] = ( u'\u2014', '') self.pub.process_programmatic_settings(settings_spec, settings_overrides, config_section) def create_xhtml(self, text, destination=None, destination_path=None, enable_exit_status=None): ''' Create xhtml for a fragment of IM dialog. We can use the source_name to store info about the message.''' self.pub.set_source(text, None) self.pub.set_destination(destination, destination_path) output = self.pub.publish(enable_exit_status=enable_exit_status) # kludge until we can get docutils to stop generating (rare) # entities return u'\u00a0'.join(self.pub.writer.parts['fragment'].strip().split( ' '))
def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document """Parse a document and convert to doctree.""" input_class = app.registry.get_source_input(filename) reader = SphinxStandaloneReader(app) source = input_class(app, env, source=None, source_path=filename, encoding=env.config.source_encoding) parser = app.registry.create_source_parser(app, filename) pub = Publisher(reader=reader, parser=parser, writer=SphinxDummyWriter(), source_class=SphinxDummySourceClass, destination=NullOutput()) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source, filename) pub.publish() return pub.document
def aoeuaoeuaoeu_docs(f): ''' I couldn't figure out how to use the docutils docstring parser, so I wrote my own. Can somebody show me the right way to do this? ''' raise NotImplementedError from docutils.core import Publisher from docutils.io import StringInput pub = Publisher(None, None, None, settings = settings, source_class = StringInput, destination_class = destination_class) pub.set_components('standalone', 'restructuredtext', 'pseudoxml') pub.process_programmatic_settings( settings_spec, settings_overrides, config_section) pub.set_source(f.__doc__, f.__name__) pub.set_destination(None, f.__name__) output = pub.publish(enable_exit_status = False) return output, pub return publish_parts(f.__doc__, source_class = StringInput, source_path = f.__name__)
def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, str) -> nodes.document """Parse a document and convert to doctree.""" # set up error_handler for the target document error_handler = UnicodeDecodeErrorHandler(env.docname) codecs.register_error('sphinx', error_handler) # type: ignore reader = SphinxStandaloneReader(app) filetype = get_filetype(app.config.source_suffix, filename) parser = app.registry.create_source_parser(app, filetype) if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == (): # a workaround for recommonmark # If recommonmark.AutoStrictify is enabled, the parser invokes reST parser # internally. But recommonmark-0.4.0 does not provide settings_spec for reST # parser. As a workaround, this copies settings_spec for RSTParser to the # CommonMarkParser. parser.settings_spec = RSTParser.settings_spec input_class = app.registry.get_source_input(filetype) if input_class: # Sphinx-1.8 style source = input_class(app, env, source=None, source_path=filename, # type: ignore encoding=env.config.source_encoding) pub = Publisher(reader=reader, # type: ignore parser=parser, writer=SphinxDummyWriter(), source_class=SphinxDummySourceClass, destination=NullOutput()) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source, filename) else: # Sphinx-2.0 style pub = Publisher(reader=reader, parser=parser, writer=SphinxDummyWriter(), source_class=SphinxFileInput, destination=NullOutput()) pub.process_programmatic_settings(None, env.settings, None) pub.set_source(source_path=filename) pub.publish() return pub.document
def parts_from_doctree(document, destination_path=None, writer=Writer(), writer_name='html4css2', settings=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=None): """ A version of :func:`docutils.core.publish_from_doctree` that supplies the "parts" rather than a regular string representation. """ reader = Reader(parser_name='null') pub = Publisher(reader, None, writer, source=io.DocTreeInput(document), destination_class=io.StringOutput, settings=settings) if not writer and writer_name: pub.set_writer(writer_name) pub.process_programmatic_settings( settings_spec, settings_overrides, config_section) pub.set_destination(None, destination_path) output = pub.publish() return pub.writer.parts
def parts_from_doctree(document, destination_path=None, writer=None, writer_name='pseudoxml', settings=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=None): """ Set up & run a `Publisher` to render from an existing document tree data structure, for programmatic use with string I/O. Return the encoded string output. Note that document.settings is overridden; if you want to use the settings of the original `document`, pass settings=document.settings. Also, new document.transformer and document.reporter objects are generated. For encoded string output, be sure to set the 'output_encoding' setting to the desired encoding. Set it to 'unicode' for unencoded Unicode string output. Here's one way:: publish_from_doctree( ..., settings_overrides={'output_encoding': 'unicode'}) Parameters: `document` is a `docutils.nodes.document` object, an existing document tree. Other parameters: see `publish_programmatically`. """ reader = docutils.readers.doctree.Reader(parser_name='null') pub = Publisher(reader, None, writer, source=io.DocTreeInput(document), destination_class=io.StringOutput, settings=settings) if not writer and writer_name: pub.set_writer(writer_name) pub.process_programmatic_settings( settings_spec, settings_overrides, config_section) pub.set_destination(None, destination_path) pub.publish(enable_exit_status=enable_exit_status) return pub.writer.parts
def sandbox_partial_builder(doctree, env): env.process_downloads(env.docname, doctree) for domain in env.domains.values(): domain.process_doc(env, env.docname, doctree) env.resolve_references(doctree, env.docname, env.app.builder) if not hasattr(env.app.builder, 'dlpath'): # FIXME: builders.html.StandaloneHTMLBuilder.write_doc で設定される属性のため、この時点では存在しない env.app.builder.dlpath = '_downloads' env.app.builder.fignumbers = env.toc_secnumbers.get(env.docname, {}) env.app.builder.secnumbers = env.toc_secnumbers.get(env.docname, {}) pub = Publisher( reader=SandboxDoctreeReader(), writer=Writer(env.app.builder), source=DocTreeInput(doctree), destination_class=StringOutput, ) pub.set_components(None, 'restructuredtext', None) defaults = env.settings.copy() defaults['output_encoding'] = 'unicode' pub.process_programmatic_settings(None, defaults, None) pub.set_destination(None, None) out = pub.publish(enable_exit_status=False) return pub.writer.parts['fragment']
def __init__(self, app): # After much digging through source code, this is how we emulate Sphinx's builtin reST parsing state # https://github.com/sphinx-doc/sphinx/blob/68cc0f7e94f360a2c62ebcb761f8096e04ebf07f/sphinx/io.py#L204 # Here we're bypassing the RST Parser, and just doing the relevant code ops parser = app.registry.create_source_parser(app, "restructuredtext") # autosummary uses tab width 8; not set by publisher/env for some reason settings = dict(app.env.settings) if "tab_width" not in settings: settings["tab_width"] = 8 p2 = Publisher() p2.process_programmatic_settings(None, settings, None) document = new_document('dummy_kissapi_source', p2.settings) # (source path, settings) document.reporter = NullReporter() state_machine = RSTStateMachine( state_classes, 'Body') # (state machine classes, initial state) # needed to set various self.[attr] values of state_machine state_machine.run([""], document) # (input lines, document) # the directive attrs that are needed self.state = state_machine.get_state() self.env = app.env self.lineno = 0
def read_initial_doctree(filename: pathlib.Path, logger: logging.Logger) -> Optional[nodes.document]: """Parse the given reStructuredText file into its "initial" doctree. An "initial" doctree can be thought of as the abstract syntax tree of a reStructuredText document. This method disables all role and directives from being executed, instead they are replaced with nodes that simply represent that they exist. Parameters ---------- filename Returns the doctree that corresponds with the given file. logger Logger to log debug info to. """ parser = Parser() if filename.suffix.replace(".", "") not in parser.supported: return None logger.debug("Getting initial doctree for: '%s'", filename) with disable_roles_and_directives(): publisher = Publisher( reader=InitialDoctreeReader(logger), parser=parser, writer=DummyWriter(), source_class=FileInput, destination=NullOutput(), ) publisher.process_programmatic_settings(None, default_settings, None) publisher.set_source(source_path=str(filename)) publisher.publish() return publisher.document
def _get_publisher(self, source_path, translator_class=BlogHTMLTranslator): extra_params = { "initial_header_level": "2", "syntax_highlight": "short", "input_encoding": "utf-8", "exit_status_level": 2, "language_code": self._language_code, "halt_level": 2, "traceback": True, "warning_stream": StringIO(), "embed_stylesheet": False, } user_params = self.settings.get("DOCUTILS_SETTINGS") if user_params: extra_params.update(user_params) pub = Publisher(writer=self.writer_class(), destination_class=StringOutput) pub.set_components("standalone", "restructuredtext", "html") pub.writer.translator_class = translator_class pub.process_programmatic_settings(None, extra_params, None) pub.set_source(source_path=source_path) pub.publish() return pub
class _PydocParser: def __init__(self): # Set up the instance we'll be using to render docstrings. self.errors = [] self.writer = _DocumentPseudoWriter() self.publisher = Publisher(_EpydocReader(self.errors), writer=self.writer, source_class=io.StringInput) self.publisher.set_components('standalone', 'restructuredtext', 'pseudoxml') settings_overrides={ 'report_level':10000, 'halt_level':10000, 'warning_stream':None, } self.publisher.process_programmatic_settings(None, settings_overrides, None) self.publisher.set_destination() def parse_docstring(self, docstring, errors): """Parse a docstring for eventual transformation into HTML This function is a replacement for parse_docstring from epydoc.markup.restructuredtext.parse_docstring. This function reuses the Publisher instance while the original did not. Using This function yields significantly faster WADL generation for complex systems. """ # Clear any errors from previous calls. del self.errors[:] self.publisher.set_source(docstring, None) self.publisher.publish() # Move any errors into the caller-provided list. errors[:] = self.errors[:] return ParsedRstDocstring(self.writer.document)
def write_doc(self, docname, doctree): outfilename = path.join(self.outdir, self.file_transform(docname)) logger.debug(outfilename) reader = docutils.readers.doctree.Reader(parser_name='null') #Note, want to use our OdtWriter class here - but it doesn't work yet writer = Writer() pub = Publisher(reader, None, writer, settings=None, source=io.DocTreeInput(doctree), destination_class=io.BinaryFileOutput) settings_spec = None settings_overrides = {'output_encoding': 'unicode'} config_section = None pub.process_programmatic_settings(settings_spec, settings_overrides, config_section) destination = None destination_path = outfilename pub.set_destination(destination, destination_path) output = pub.publish(enable_exit_status=False)
def get_doctree(path, **kwargs): """ Obtain a Sphinx doctree from the RST file at ``path``. Performs no Releases-specific processing; this code would, ideally, be in Sphinx itself, but things there are pretty tightly coupled. So we wrote this. Any additional kwargs are passed unmodified into an internal `make_app` call. :param str path: A relative or absolute file path string. :returns: A two-tuple of the generated ``sphinx.application.Sphinx`` app and the doctree (a ``docutils.document`` object). .. versionchanged:: 1.6 Added support for passing kwargs to `make_app`. """ root, filename = os.path.split(path) docname, _ = os.path.splitext(filename) # TODO: this only works for top level changelog files (i.e. ones where # their dirname is the project/doc root) app = make_app(srcdir=root, **kwargs) # Create & init a BuildEnvironment. Mm, tasty side effects. app._init_env(freshenv=True) env = app.env # More arity/API changes: Sphinx 1.3/1.4-ish require one to pass in the app # obj in BuildEnvironment.update(); modern Sphinx performs that inside # Application._init_env() (which we just called above) and so that kwarg is # removed from update(). EAFP. kwargs = dict( config=app.config, srcdir=root, doctreedir=app.doctreedir, app=app, ) try: env.update(**kwargs) except TypeError: # Assume newer Sphinx w/o an app= kwarg del kwargs['app'] env.update(**kwargs) # Code taken from sphinx.environment.read_doc; easier to manually call # it with a working Environment object, instead of doing more random crap # to trick the higher up build system into thinking our single changelog # document was "updated". env.temp_data['docname'] = docname env.app = app # NOTE: SphinxStandaloneReader API changed in 1.4 :( reader_kwargs = { 'app': app, 'parsers': env.config.source_parsers, } if sphinx.version_info[:2] < (1, 4): del reader_kwargs['app'] # This monkeypatches (!!!) docutils to 'inject' all registered Sphinx # domains' roles & so forth. Without this, rendering the doctree lacks # almost all Sphinx magic, including things like :ref: and :doc:! with sphinx_domains(env): reader = SphinxStandaloneReader(**reader_kwargs) pub = Publisher(reader=reader, writer=SphinxDummyWriter(), destination_class=NullOutput) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, env.settings, None) # NOTE: docname derived higher up, from our given path src_path = env.doc2path(docname) source = SphinxFileInput( app, env, source=None, source_path=src_path, encoding=env.config.source_encoding, ) pub.source = source pub.settings._source = src_path pub.set_destination(None, None) pub.publish() return app, pub.document
def env_updated(app, env): config = app.builder.config doctree = env.get_doctree(config.master_doc) from sphinx import addnodes toctrees = [] for toctreenode in doctree.traverse(addnodes.toctree): toctree = env.resolve_toctree(config.master_doc, app.builder, toctreenode, prune = False, includehidden = True, maxdepth = 0, collapse = False) toctrees.append(toctree) if not toctrees: toc = None else: toc = toctrees[0] for toctree in toctrees[1:]: toc.extend(toctree.children) # toc = env.get_toctree_for(config.master_doc, app.builder, False) node = toc doc = new_document(b('<partial node>')) doc.append(node) pub = Publisher( source_class = DocTreeInput, destination_class=StringOutput) pub.set_components('standalone', 'restructuredtext', 'pseudoxml') pub.reader = DoctreeReader() pub.writer = Writer() pub.writer.format = 'pseudoxml' pub.process_programmatic_settings( None, {'output_encoding': 'unicode'}, None) pub.set_source(doc, None) pub.set_destination(None, None) pub.publish() import xml.etree.cElementTree as ET from cStringIO import StringIO #out = re.sub(r'^<!DOCTYPE[^>]*>\s*', '<?xml version="1.0"?>', pub.writer.output) out = pub.writer.output.encode('utf-8').replace(' encoding="unicode"', ' encoding="utf-8"') #pprint.pprint(out) doctree = ET.fromstring(out) #pprint.pprint(doctree) #pprint.pprint(dir(doctree)) if hasattr(doctree, 'getroot'): doctree = doctree.getroot() #root = doctree.getroot() fuzzy_find_entries = [] docs = {} def indexentries(entry, links, cap = ''): if links: fuzzy_find_entries.append(dict( href = links[0][1], name = entry, path = "index/%s/%s" % (char,cap), info = "INDEX", detail = '', )) for i, (ismain, link) in enumerate(links[1:], start=1): doclink = link.split('#', 1)[0] docname = docs.get(doclink, i) fuzzy_find_entries.append(dict( href = link, name = "%s (%s)" % (entry, docname), path = "index/%s/%s" % (char, cap), info = "INDEX", detail = '', )) return '' else: return entry if app.config.html_findanything_add_topics: if hasattr(doctree, 'iter'): references = doctree.iter('reference') else: references = doctree.getiterator('reference') # fuzzy_find_entries.append(dict( # href = link, # name = entry, # path = "index/%s/" % char, # info = "INDEX", # detail = '', # )) refset = set() for ref in references: refuri = ref.attrib['refuri'] docs[refuri] = ref.text path = "/" if "/" in refuri: path = refuri.rsplit('/', 1)[0]+"/" if '#' in refuri: docname = docs.get(refuri.split('#', 1)[0]) if docname: path += docname + "/" info = 'SECTION' else: info = 'PAGE' e = dict( href = ref.attrib['refuri'], name = ref.text, info = info, path = path, detail = '', ) refid = "%(href)s^%(path)s^%(name)s^%(info)s" % e if refid in refset: continue refset.add(refid) fuzzy_find_entries.append(e) if app.config.html_findanything_add_indexentries: genindex = env.create_index(app.builder) for char,char_list in genindex: for entry, (links, subitems) in char_list: cap = indexentries(entry, links) if subitems: if cap: cap += '/' for subname, sublinks in subitems: indexentries(subname, sublinks, cap=cap) s = json.dumps(fuzzy_find_entries) static_dir = os.path.join(app.builder.outdir, '_static') if not os.path.exists(static_dir): os.makedirs(static_dir) with open(os.path.join(static_dir, 'fuzzyindex.js'), 'wb') as f: f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_ENTRIES = %s;" % s); if app.config.html_findanything_use_cached_hits: f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_USE_CACHED_HITS = true;") f.write("DOCUMENTATION_OPTIONS.FIND_ANYTHING_WIDTH = '%s';" % app.config.html_findanything_width);
def create_toc(doctree, depth=9223372036854775807, writer_name='html', exclude_first_section=True, href_prefix=None, id_prefix='toc-ref-'): """ Create a Table of Contents (TOC) from the given doctree Returns: (docutils.core.Publisher instance, output string) `writer_name`: represents a reST writer name and determines the type of output returned. Example: pub = blazeutils.rst.rst2pub(toc_rst) pub, html_output = blazeutils.rst.create_toc(pub.document) # a full HTML document (probably not what you want most of the time) print html_output # just the TOC print pub.writer.parts['body'] """ # copy the doctree since Content alters some settings on the original # document doctree = doctree.deepcopy() # we want to be able to customize ids to avoid clashes if needed doctree.settings.auto_id_prefix = id_prefix details = { 'depth': depth, } # Assuming the document has one primary heading and then sub-sections, we # want to be able to give just the sub-sections startnode = None if exclude_first_section: nodes = doctree.traverse(docutils.nodes.section) if nodes: startnode = nodes[0] # use the Contents transform to build the TOC node structure from the # document c = Contents(doctree) # this startnode isn't really used as the start node, its only used for # to pull settings from c.startnode = BlankObject(details=details) # since this toc is detached from the rest of the document, we don't want # backlinks c.backlinks = 'none' # build the nodes toc_nodes = c.build_contents(startnode or doctree) # create a new document with the new nodes toc_doc = new_document(None) toc_doc += toc_nodes # fix fragements that reference the same page if href_prefix: prefix_refids(toc_doc, href_prefix) # setup a publisher and publish from the TOC document reader = docutils.readers.doctree.Reader(parser_name='null') pub = Publisher( reader, None, None, source=io.DocTreeInput(toc_doc), destination_class=io.StringOutput ) pub.set_writer(writer_name) final_settings_overrides = default_rst_opts.copy() pub.process_programmatic_settings( None, final_settings_overrides, None) output = pub.publish() return pub, output
def read_doc(self, docname, app=None): # type: (unicode, Sphinx) -> None """Parse a file and add/update inventory entries for the doctree.""" self.temp_data['docname'] = docname # defaults to the global default, but can be re-set in a document self.temp_data['default_domain'] = \ self.domains.get(self.config.primary_domain) self.settings['input_encoding'] = self.config.source_encoding self.settings['trim_footnote_reference_space'] = \ self.config.trim_footnote_reference_space self.settings['gettext_compact'] = self.config.gettext_compact docutilsconf = path.join(self.srcdir, 'docutils.conf') # read docutils.conf from source dir, not from current dir OptionParser.standard_config_files[1] = docutilsconf if path.isfile(docutilsconf): self.note_dependency(docutilsconf) with sphinx_domains(self): if self.config.default_role: role_fn, messages = roles.role(self.config.default_role, english, 0, dummy_reporter) if role_fn: roles._roles[''] = role_fn else: logger.warning('default role %s not found', self.config.default_role, location=docname) codecs.register_error('sphinx', self.warn_and_replace) # type: ignore # publish manually reader = SphinxStandaloneReader(self.app, parsers=self.config.source_parsers) pub = Publisher(reader=reader, writer=SphinxDummyWriter(), destination_class=NullOutput) pub.set_components(None, 'restructuredtext', None) pub.process_programmatic_settings(None, self.settings, None) src_path = self.doc2path(docname) source = SphinxFileInput(app, self, source=None, source_path=src_path, encoding=self.config.source_encoding) pub.source = source pub.settings._source = src_path pub.set_destination(None, None) pub.publish() doctree = pub.document # post-processing for domain in itervalues(self.domains): domain.process_doc(self, docname, doctree) # allow extension-specific post-processing if app: app.emit('doctree-read', doctree) # store time of reading, for outdated files detection # (Some filesystems have coarse timestamp resolution; # therefore time.time() can be older than filesystem's timestamp. # For example, FAT32 has 2sec timestamp resolution.) self.all_docs[docname] = max( time.time(), path.getmtime(self.doc2path(docname))) if self.versioning_condition: old_doctree = None if self.versioning_compare: # get old doctree try: with open(self.doc2path(docname, self.doctreedir, '.doctree'), 'rb') as f: old_doctree = pickle.load(f) except EnvironmentError: pass # add uids for versioning if not self.versioning_compare or old_doctree is None: list(add_uids(doctree, self.versioning_condition)) else: list(merge_doctrees( old_doctree, doctree, self.versioning_condition)) # make it picklable doctree.reporter = None doctree.transformer = None doctree.settings.warning_stream = None doctree.settings.env = None doctree.settings.record_dependencies = None # cleanup self.temp_data.clear() self.ref_context.clear() roles._roles.pop('', None) # if a document has set a local default role # save the parsed doctree doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree') ensuredir(path.dirname(doctree_filename)) with open(doctree_filename, 'wb') as f: pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
from docutils.core import publish_cmdline, default_description,\ publish_doctree,\ Publisher from dotmpe.du.ext.writer.formresults import Writer as FormResultsWriter #publish_cmdline(reader=FormReader(), writer_name='pseudoxml')#writer=xhtmlform.Writer()) source_path = sys.argv[1] source = open(source_path) #doctree = publish_doctree(source, reader=FormReader())#, writer_name='pseudoxml') #print doctree pub = Publisher(FormReader(), writer=FormResultsWriter(), source_class=io.FileInput, destination_class=io.StringOutput)#, parser, writer, settings=settings, # source_class=source_class, # destination_class=destination_class) pub.set_components(None, 'rst', 'pseudoxml') pub.process_programmatic_settings(None, None, None) # settings_spec, settings_overrides, config_section) pub.set_source(source, source_path) pub.set_destination(None, None)#destination, destination_path) output = pub.publish(enable_exit_status=False) #print pub.document.form_processor.fields #print pub.document.form_processor.messages #print pub.document.form_processor.values print output
class RstPublisher: ############################################## def __init__(self): self._publisher = Publisher(source_class=StringInput, destination_class=StringOutput) self._publisher.set_components('standalone', 'restructuredtext', 'html') self._publisher.writer.translator_class = MyHTMLTranslator self._publisher.process_programmatic_settings(None, extra_params, None) ############################################## def init_plugins(self, plugins): site.addsitedir(str(PurePath(__file__).parent / 'RstPlugins')) for plugin in plugins: print('Register {}'.format(plugin)) plugin = __import__(plugin, globals(), locals(), str('module')) plugin.register(self) ############################################## def register_directive(self, directive_name, directive_class): directives.register_directive(directive_name, directive_class) ############################################## def register_role(self, role_name, role_class): roles.register_local_role(role_name, role_class) ############################################## def register_node(self, node_class, visit, depart): print(node_class.__name__, visit, depart) setattr(MyHTMLTranslator, 'visit_' + node_class.__name__, visit) setattr(MyHTMLTranslator, 'depart_' + node_class.__name__, depart) ############################################## def publish(self, source, header_level=None, report_level=None): settings_overrides = DOCUTILS_RENDERER_SETTINGS.copy() if header_level is not None: # starts from 1 settings_overrides['initial_header_level'] = header_level if report_level is not None: # starts from 1 too settings_overrides['report_level'] = 0 # report_level self._publisher.set_source(source=source) self._publisher.publish(enable_exit_status=True) parts = self._publisher.writer.parts return parts[ 'html_body'] # parts['body_pre_docinfo'] + parts['fragment']
def process_targets(site, logger, source, permalink): """Process the target locations in the reST files.""" site.processing_targets = True reader = Reader() reader.l_settings = {"source": source} with open(source, "r", encoding="utf8") as in_file: data = in_file.read() pub = Publisher( reader=reader, parser=None, writer=None, settings=None, source_class=StringInput, destination_class=StringOutput, ) pub.set_components(None, "restructuredtext", "html") # Reading the file will generate output/errors that we don't care about # at this stage. The report_level = 5 means no output pub.process_programmatic_settings(settings_spec=None, settings_overrides={"report_level": 5}, config_section=None) pub.set_source(data, None) pub.set_destination(None, None) pub.publish() document = pub.document site.processing_targets = False # Code based on Sphinx std domain for name, is_explicit in document.nametypes.items(): if not is_explicit: continue labelid = document.nameids[name] if labelid is None: continue node = document.ids[labelid] if node.tagname == "target" and "refid" in node: node = document.ids.get(node["refid"]) labelid = node["names"][0] if (node.tagname == "footnote" or "refuri" in node or node.tagname.startswith("desc_")): continue if name in site.ref_targets: logger.warn( "Duplicate label {dup}, other instance in {other}".format( dup=name, other=site.ref_targets[name][0])) site.anon_ref_targets[name] = permalink, labelid def clean_astext(node): """Like node.astext(), but ignore images. Taken from sphinx.util.nodes """ node = node.deepcopy() for img in node.traverse(nodes.image): img["alt"] = "" for raw in node.traverse(nodes.raw): raw.parent.remove(raw) return node.astext() if node.tagname in ("section", "rubric"): sectname = clean_astext(node[0]) else: continue site.ref_targets[name] = permalink, labelid, sectname