from os import path from six import iteritems, text_type, BytesIO, StringIO from sphinx import package_dir from sphinx.errors import PycodeError from sphinx.pycode import nodes from sphinx.pycode.pgen2 import driver, token, tokenize, parse, literals from sphinx.util import get_module_source, detect_encoding from sphinx.util.pycompat import TextIOWrapper from sphinx.util.docstrings import prepare_docstring, prepare_commentdoc # load the Python grammar _grammarfile = path.join(package_dir, 'pycode', 'Grammar-py%d.txt' % sys.version_info[0]) pygrammar = driver.load_grammar(_grammarfile) pydriver = driver.Driver(pygrammar, convert=nodes.convert) # an object with attributes corresponding to token and symbol names class sym: pass for k, v in iteritems(pygrammar.symbol2number): setattr(sym, k, v) for k, v in iteritems(token.tok_name): setattr(sym, v, k) # a dict mapping terminal and nonterminal numbers to their names number2name = pygrammar.number2symbol.copy()
:license: BSD, see LICENSE for details. """ import re import sys from os import path from cStringIO import StringIO from sphinx.pycode import nodes from sphinx.pycode.pgen2 import driver, token, tokenize, parse, literals from sphinx.util.docstrings import prepare_docstring, prepare_commentdoc # load the Python grammar _grammarfile = path.join(path.dirname(__file__), 'Grammar.txt') pygrammar = driver.load_grammar(_grammarfile) pydriver = driver.Driver(pygrammar, convert=nodes.convert) # an object with attributes corresponding to token and symbol names class sym: pass for k, v in pygrammar.symbol2number.iteritems(): setattr(sym, k, v) for k, v in token.tok_name.iteritems(): setattr(sym, v, k) # a dict mapping terminal and nonterminal numbers to their names number2name = pygrammar.number2symbol.copy() number2name.update(token.tok_name) # a regex to recognize coding cookies