def run_cmdline(*args, **kwds):
saved_stdin = sys.stdin
saved_stdout = sys.stdout
saved_stderr = sys.stderr
if sys.version_info > (3, ):
stdin_buffer = BytesIO()
stdout_buffer = BytesIO()
stderr_buffer = BytesIO()
new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8')
new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
else:
stdin_buffer = new_stdin = sys.stdin = StringIO()
stdout_buffer = new_stdout = sys.stdout = StringIO()
stderr_buffer = new_stderr = sys.stderr = StringIO()
new_stdin.write(kwds.get('stdin', ''))
new_stdin.seek(0, 0)
try:
ret = cmdline.main(['pygmentize'] + list(args))
finally:
sys.stdin = saved_stdin
sys.stdout = saved_stdout
sys.stderr = saved_stderr
new_stdout.flush()
new_stderr.flush()
out, err = stdout_buffer.getvalue().decode('utf-8'), \
stderr_buffer.getvalue().decode('utf-8')
return (ret, out, err)
def test_lineanchors():
optdict = dict(lineanchors="foo")
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
assert re.search("<pre><span></span><a name=\"foo-1\">", html)
def test_linenos_with_startnum(self):
optdict = dict(linenos=True, linenostart=5)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html))
def test_filename():
optdict = dict(filename="test.py")
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
assert re.search("<span class=\"filename\">test.py</span><pre>", html)
def test_linenos(self):
optdict = dict(linenos=True)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html))
def test_linenos_with_startnum(self):
optdict = dict(linenos=True, linenostart=5)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html))
def test_lineanchors_with_startnum(self):
optdict = dict(lineanchors="foo", linenostart=5)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre><span></span><a name=\"foo-5\">", html))
def test_lineanchors(self):
optdict = dict(lineanchors="foo")
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre><a name=\"foo-1\">", html))
def test_linenos(self):
optdict = dict(linenos=True)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html))
def test_lineanchors(self):
optdict = dict(lineanchors="foo")
outfile = StringIO()
fmt = CodeHtmlFormatter(instance_class=type, **optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre><a name=\"foo-1\">", html))
def test_correct_output(self):
hfmt = IRCFormatter()
houtfile = StringIO()
hfmt.format(tokensource, houtfile)
self.assertEqual(u'\x0302lambda\x03 x: \x0302123\x03\n',
houtfile.getvalue())
def test_filename(self):
optdict = dict(filename="test.py")
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<span class=\"filename\">test.py</span><pre>", html))
def test_lineanchors_with_startnum(self):
optdict = dict(lineanchors="foo", linenostart=5)
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
html = outfile.getvalue()
self.assertTrue(re.search("<pre><a name=\"foo-5\">", html))
def format_rtf(self, t):
tokensource = list(TextLexer().get_tokens(t))
fmt = RtfFormatter()
buf = StringIO()
fmt.format(tokensource, buf)
result = buf.getvalue()
buf.close()
return result
def format_rtf(self, t):
tokensource = list(TextLexer().get_tokens(t))
fmt = RtfFormatter()
buf = StringIO()
fmt.format(tokensource, buf)
result = buf.getvalue()
buf.close()
return result
def test_reasonable_output(self):
out = StringIO()
TerminalFormatter().format(DEMO_TOKENS, out)
plain = strip_ansi(out.getvalue())
self.assertEqual(DEMO_TEXT.count('\n'), plain.count('\n'))
print(repr(plain))
for a, b in zip(DEMO_TEXT.splitlines(), plain.splitlines()):
self.assertEqual(a, b)
def test_reasonable_output_lineno():
out = StringIO()
TerminalFormatter(linenos=True).format(DEMO_TOKENS, out)
plain = strip_ansi(out.getvalue())
assert DEMO_TEXT.count('\n') + 1 == plain.count('\n')
print(repr(plain))
for a, b in zip(DEMO_TEXT.splitlines(), plain.splitlines()):
assert a in b
def test_incode_links(self):
# reference another method
self.noop()
this_token_source = list(PythonLexer().get_tokens(
inspect.getsource(CodeHtmlFormatterTest.test_incode_links)))
hfmt = CodeHtmlFormatter(instance_class=self.__class__, nowrap=True)
houtfile = StringIO()
hfmt.format(this_token_source, houtfile)
assert '<a href="#noop">noop</a>' in houtfile.getvalue()
def test_reasonable_output_lineno(self):
out = StringIO()
TerminalFormatter(linenos=True).format(DEMO_TOKENS, out)
plain = strip_ansi(out.getvalue())
self.assertEqual(DEMO_TEXT.count('\n') + 1, plain.count('\n'))
print(repr(plain))
for a, b in zip(DEMO_TEXT.splitlines(), plain.splitlines()):
self.assertTrue(a in b)
def run_cmdline(*args):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
new_stdout = sys.stdout = StringIO()
new_stderr = sys.stderr = StringIO()
try:
ret = cmdline_main(["pygmentize"] + list(args))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
return (ret, new_stdout.getvalue(), new_stderr.getvalue())
def test_correct_output(self):
hfmt = HtmlFormatter(nowrap=True)
houtfile = StringIO()
hfmt.format(tokensource, houtfile)
nfmt = NullFormatter()
noutfile = StringIO()
nfmt.format(tokensource, noutfile)
stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
escaped_text = escape_html(noutfile.getvalue())
self.assertEqual(stripped_html, escaped_text)
def test_ctags():
try:
import ctags
except ImportError:
# we can't check without the ctags module, but at least check the exception
assert raises(RuntimeError, HtmlFormatter, tagsfile='support/tags')
else:
# this tagfile says that test_ctags() is on line 165, even if it isn't
# anymore in the actual source
fmt = HtmlFormatter(tagsfile='support/tags', lineanchors='L',
tagurlformat='%(fname)s%(fext)s')
outfile = StringIO()
fmt.format(tokensource, outfile)
assert '<a href="test_html_formatter.py#L-165">test_ctags</a>' \
in outfile.getvalue()
def test_ctags(self):
try:
import ctags
except ImportError:
# we can't check without the ctags module, but at least check the exception
self.assertRaises(RuntimeError, HtmlFormatter, tagsfile='support/tags')
else:
# this tagfile says that test_ctags() is on line 165, even if it isn't
# anymore in the actual source
fmt = HtmlFormatter(tagsfile='support/tags', lineanchors='L',
tagurlformat='%(fname)s%(fext)s')
outfile = StringIO()
fmt.format(tokensource, outfile)
self.assertTrue('<a href="test_html_formatter.py#L-165">test_ctags</a>'
in outfile.getvalue())
def test_formatter_public_api():
ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
out = StringIO()
# test that every formatter class has the correct public API
def verify(formatter, info):
assert len(info) == 4
assert info[0], "missing formatter name"
assert info[1], "missing formatter aliases"
assert info[3], "missing formatter docstring"
if formatter.name == 'Raw tokens':
# will not work with Unicode output file
return
try:
inst = formatter(opt1="val1")
except (ImportError, FontNotFound):
return
try:
inst.get_style_defs()
except NotImplementedError:
# may be raised by formatters for which it doesn't make sense
pass
inst.format(ts, out)
for formatter, info in formatters.FORMATTERS.items():
yield verify, formatter, info
def test_formatter_public_api(cls):
# test that every formatter class has the correct public API
ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
string_out = StringIO()
bytes_out = BytesIO()
info = formatters.FORMATTERS[cls.__name__]
assert len(info) == 5
assert info[1], "missing formatter name"
assert info[2], "missing formatter aliases"
assert info[4], "missing formatter docstring"
try:
inst = cls(opt1="val1")
except (ImportError, FontNotFound) as e:
pytest.skip(str(e))
try:
inst.get_style_defs()
except NotImplementedError:
# may be raised by formatters for which it doesn't make sense
pass
if cls.unicodeoutput:
inst.format(ts, string_out)
else:
inst.format(ts, bytes_out)
def test_formatter_public_api():
# test that every formatter class has the correct public API
ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
string_out = StringIO()
bytes_out = BytesIO()
def verify(formatter):
info = formatters.FORMATTERS[formatter.__name__]
assert len(info) == 5
assert info[1], "missing formatter name"
assert info[2], "missing formatter aliases"
assert info[4], "missing formatter docstring"
try:
inst = formatter(opt1="val1")
except (ImportError, FontNotFound):
raise support.SkipTest
try:
inst.get_style_defs()
except NotImplementedError:
# may be raised by formatters for which it doesn't make sense
pass
if formatter.unicodeoutput:
inst.format(ts, string_out)
else:
inst.format(ts, bytes_out)
for name in formatters.FORMATTERS:
formatter = getattr(formatters, name)
yield verify, formatter
def test_all_options(self):
for optdict in [dict(nowrap=True),
dict(linenos=True),
dict(linenos=True, full=True),
dict(linenos=True, full=True, noclasses=True)]:
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
def run_cmdline(*args):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
if sys.version_info > (3,):
stdout_buffer = BytesIO()
stderr_buffer = BytesIO()
new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
else:
stdout_buffer = new_stdout = sys.stdout = StringIO()
stderr_buffer = new_stderr = sys.stderr = StringIO()
try:
ret = cmdline.main(["pygmentize"] + list(args))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
new_stdout.flush()
new_stderr.flush()
out, err = stdout_buffer.getvalue().decode('utf-8'), \
stderr_buffer.getvalue().decode('utf-8')
return (ret, out, err)
def test_correct_output(self):
hfmt = HtmlFormatter(nowrap=True)
houtfile = StringIO()
hfmt.format(tokensource, houtfile)
nfmt = NullFormatter()
noutfile = StringIO()
nfmt.format(tokensource, noutfile)
stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
escaped_text = escape_html(noutfile.getvalue())
self.assertEqual(stripped_html, escaped_text)
def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin
"""
Format a tokenlist ``tokens`` with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
try:
if not outfile:
realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
formatter.format(tokens, realoutfile)
return realoutfile.getvalue()
else:
formatter.format(tokens, outfile)
except TypeError as err:
if (isinstance(err.args[0], str) and
('unbound method format' in err.args[0] or
'missing 1 required positional argument' in err.args[0])):
raise TypeError('format() argument must be a formatter instance, '
'not a class')
raise
def format(tokens, formatter, outfile=None):
"""
Format a tokenlist ``tokens`` with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
try:
if not outfile:
#print formatter, 'using', formatter.encoding
realoutfile = formatter.encoding and BytesIO() or StringIO()
formatter.format(tokens, realoutfile)
return realoutfile.getvalue()
else:
formatter.format(tokens, outfile)
except TypeError, err:
if isinstance(err.args[0], str) and \
'unbound method format' in err.args[0]:
raise TypeError('format() argument must be a formatter instance, '
'not a class')
raise
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(r'\begin{Verbatim}[commandchars=\\\{\}')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(',numbers=left' +
(start and ',firstnumber=%d' % start or '') +
(step and ',stepnumber=%d' % step or ''))
if self.mathescape or self.texcomments:
outfile.write(r',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
if self.verboptions:
outfile.write(',' + self.verboptions)
outfile.write(']\n')
for ttype, value in tokensource:
if ttype in Token.Comment:
if self.texcomments:
# Try to guess comment starting lexeme and escape it ...
start = value[0:1]
for i in xrange(1, len(value)):
if start[0] != value[i]:
break
start += value[i]
value = value[len(start):]
start = escape_tex(start, self.commandprefix)
# ... but do not escape inside comment.
value = start + value
elif self.mathescape:
# Only escape parts not inside a math environment.
parts = value.split('$')
in_math = False
for i, part in enumerate(parts):
if not in_math:
parts[i] = escape_tex(part, self.commandprefix)
in_math = not in_math
value = '$'.join(parts)
else:
value = escape_tex(value, self.commandprefix)
else:
value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write('\\end{Verbatim}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = self.encoding or 'latin1',
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(ur'\begin{Verbatim}[commandchars=\\\{\}')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(u',numbers=left' +
(start and u',firstnumber=%d' % start or u'') +
(step and u',stepnumber=%d' % step or u''))
if self.mathescape or self.texcomments:
outfile.write(ur',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
if self.verboptions:
outfile.write(u',' + self.verboptions)
outfile.write(u']\n')
for ttype, value in tokensource:
if ttype in Token.Comment:
if self.texcomments:
# Try to guess comment starting lexeme and escape it ...
start = value[0:1]
for i in xrange(1, len(value)):
if start[0] != value[i]:
break
start += value[i]
value = value[len(start):]
start = escape_tex(start, self.commandprefix)
# ... but do not escape inside comment.
value = start + value
elif self.mathescape:
# Only escape parts not inside a math environment.
parts = value.split('$')
in_math = False
for i, part in enumerate(parts):
if not in_math:
parts[i] = escape_tex(part, self.commandprefix)
in_math = not in_math
value = '$'.join(parts)
else:
value = escape_tex(value, self.commandprefix)
else:
value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write(u'\\end{Verbatim}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = self.encoding or 'latin1',
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
def test_correct_output():
hfmt = IRCFormatter()
houtfile = StringIO()
hfmt.format(tokensource, houtfile)
assert u'\x0302lambda\x03 x: \x0302123\x03\n' == houtfile.getvalue()
def _wrap_tablelinenos(self, inner):
dummyoutfile = StringIO()
lncount = 0
for t, line in inner:
if t:
lncount += 1
dummyoutfile.write(line)
fl = self.linenostart
mw = len(str(lncount + fl - 1))
sp = self.linenospecial
st = self.linenostep
la = self.lineanchors
aln = self.anchorlinenos
nocls = self.noclasses
if sp:
lines = []
for i in range(fl, fl + lncount):
if i % st == 0:
if i % sp == 0:
if aln:
lines.append(
'<a href="#%s-%d" class="special">%*d</a>' %
(la, i, mw, i))
else:
lines.append('<span class="special">%*d</span>' %
(mw, i))
else:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' %
(la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
else:
lines = []
for i in range(fl, fl + lncount):
if i % st == 0:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' %
(la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
# in case you wonder about the seemingly redundant <div> here: since the
# content in the other cell also is wrapped in a div, some browsers in
# some configurations seem to mess up the formatting...
if nocls:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td><div class="linenodiv" '
'style="background-color: #f0f0f0; padding-right: 10px">'
'<pre style="line-height: 125%">' + ls +
'</pre></div></td><td class="code">')
else:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td class="linenos"><div class="linenodiv"><pre>' +
ls + '</pre></div></td><td class="code">')
yield 0, dummyoutfile.getvalue()
yield 0, '</td></tr></table>'
def _wrap_tablelinenos(self, inner):
dummyoutfile = StringIO()
lncount = 0
for t, line in inner:
if t:
lncount += 1
dummyoutfile.write(line)
fl = self.linenostart
mw = len(str(lncount + fl - 1))
sp = self.linenospecial
st = self.linenostep
la = self.lineanchors
aln = self.anchorlinenos
nocls = self.noclasses
if sp:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if i % sp == 0:
if aln:
lines.append('<a href="#%s-%d" class="special">%*d</a>' %
(la, i, mw, i))
else:
lines.append('<span class="special">%*d</span>' % (mw, i))
else:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
else:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
yield 0, ('<table class="%stable" data-filename="%s">' % (self.cssclass, self.filename))
lineno = fl
for lineno_html, code in itertools.izip(ls.split('\n'), dummyoutfile.getvalue().split('\n')):
if nocls:
yield 0, ('<tr><td><pre style="line-height: 125%">' +
lineno_html + '</pre></td><td class="code">' +
self._wrap_code_line(code) + '</td></tr>')
else:
yield 0, ('<tr><td class="linenos"><pre>' +
lineno_html + '</pre></td><td class="code">' +
self._wrap_code_line(code) + '</td></tr>')
if lineno in self.comments:
yield 0, self.render_diff_comment(lineno, self.comments[lineno])
lineno += 1
yield 0, '</table>'
def _wrap_tablelinenos(self, inner):
dummyoutfile = StringIO()
lncount = 0
for t, line in inner:
if t:
lncount += 1
dummyoutfile.write(line)
fl = self.linenostart
mw = len(str(lncount + fl - 1))
sp = self.linenospecial
st = self.linenostep
la = self.lineanchors
aln = self.anchorlinenos
nocls = self.noclasses
if sp:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if i % sp == 0:
if aln:
lines.append('<a href="#%s-%d" class="special">%*d</a>' %
(la, i, mw, i))
else:
lines.append('<span class="special">%*d</span>' % (mw, i))
else:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
else:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
# in case you wonder about the seemingly redundant <div> here: since the
# content in the other cell also is wrapped in a div, some browsers in
# some configurations seem to mess up the formatting...
if nocls:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td><div class="linenodiv" '
'style="background-color: #f0f0f0; padding-right: 10px">'
'<pre style="line-height: 125%">' +
ls + '</pre></div></td><td class="code">')
else:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td class="linenos"><div class="linenodiv"><pre>' +
ls + '</pre></div></td><td class="code">')
yield 0, dummyoutfile.getvalue()
yield 0, '</td></tr></table>'
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(r'\begin{lstlisting}[language=,breaklines=true,escapeinside={(*@}{@*)},tabsize=4,framesep=0pt,xleftmargin=\FrameSep,xrightmargin=\FrameSep,frame=none,backgroundcolor=,fillcolor=')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(',numbers=left' +
(start and ',firstnumber=%d' % start or '') +
(step and ',stepnumber=%d' % step or ''))
if self.mathescape:
outfile.write(r',mathescape=true')
if self.texcomments:
outfile.write(r',texcl=true')
# Switching to lstlisting broke these.
if self.verboptions:
outfile.write(',' + self.verboptions)
outfile.write(']\n')
for ttype, value in tokensource:
# if ttype in Token.Comment:
# if self.texcomments:
# # Try to guess comment starting lexeme and escape it ...
# start = value[0:1]
# for i in xrange(1, len(value)):
# if start[0] != value[i]:
# break
# start += value[i]
#
# value = value[len(start):]
# start = escape_tex(start, self.commandprefix)
#
# # ... but do not escape inside comment.
# value = start + value
# elif self.mathescape:
# # Only escape parts not inside a math environment.
# parts = value.split('$')
# in_math = False
# for i, part in enumerate(parts):
# if not in_math:
# parts[i] = escape_tex(part, self.commandprefix)
# in_math = not in_math
# value = '$'.join(parts)
# else:
# value = escape_tex(value, self.commandprefix)
# else:
# value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("(*@\\%s{%s}{%s}@*)" % (cp, styleval, escape_tex(line)))
outfile.write('\n')
if spl[-1]:
outfile.write("(*@\\%s{%s}{%s}@*)" % (cp, styleval, escape_tex(spl[-1])))
else:
outfile.write(value)
outfile.write('\\end{lstlisting}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = self.encoding or 'utf-8',
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(u'{\\tt')
for ttype, value in tokensource:
if ttype in Token.Comment:
if self.texcomments:
# Try to guess comment starting lexeme and escape it ...
start = value[0:1]
for i in xrange(1, len(value)):
if start[0] != value[i]:
break
start += value[i]
value = value[len(start):]
start = escape_tex(start, cp)
# ... but do not escape inside comment.
value = start + value
elif self.mathescape:
# Only escape parts not inside a math environment.
parts = value.split('$')
in_math = False
for i, part in enumerate(parts):
if not in_math:
parts[i] = escape_tex(part, cp)
in_math = not in_math
value = '$'.join(parts)
elif self.escapeinside:
text = value
value = ''
while text:
a, sep1, text = text.partition(self.left)
if sep1:
b, sep2, text = text.partition(self.right)
if sep2:
value += escape_tex(a, cp) + b
else:
value += escape_tex(a + sep1 + b, cp)
else:
value += escape_tex(a, cp)
else:
value = escape_tex(value, cp)
elif ttype not in Token.Escape:
value = escape_tex(value, cp)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = list(reversed(styles))[0]
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("\\%s%s{%s}" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("\\%s%s{%s}" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write(u'}\n')
if self.full:
encoding = self.encoding or 'utf8'
# map known existings encodings from LaTeX distribution
encoding = {
'utf_8': 'utf8',
'latin_1': 'latin1',
'iso_8859_1': 'latin1',
}.get(encoding.replace('-', '_'), encoding)
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = encoding,
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(r'\begin{Verbatim}[commandchars=@\[\]')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(',numbers=left' +
(start and ',firstnumber=%d' % start or '') +
(step and ',stepnumber=%d' % step or ''))
if self.verboptions:
outfile.write(',' + self.verboptions)
outfile.write(']\n')
for ttype, value in tokensource:
value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("@%s[%s][%s]" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("@%s[%s][%s]" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write('\\end{Verbatim}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = self.encoding or 'latin1',
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
def test_correct_output(self):
hfmt = IRCFormatter()
houtfile = StringIO()
hfmt.format(tokensource, houtfile)
self.assertEqual(u'\x0302lambda\x03 x: \x0302123\x03\n', houtfile.getvalue())
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(
r'\begin{lstlisting}[language=,breaklines=true,escapeinside={(*@}{@*)},tabsize=4,framesep=0pt,xleftmargin=\FrameSep,xrightmargin=\FrameSep,frame=none,backgroundcolor=,fillcolor='
)
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(',numbers=left' +
(start and ',firstnumber=%d' % start or '') +
(step and ',stepnumber=%d' % step or ''))
if self.mathescape:
outfile.write(r',mathescape=true')
if self.texcomments:
outfile.write(r',texcl=true')
# Switching to lstlisting broke these.
if self.verboptions:
outfile.write(',' + self.verboptions)
outfile.write(']\n')
for ttype, value in tokensource:
# if ttype in Token.Comment:
# if self.texcomments:
# # Try to guess comment starting lexeme and escape it ...
# start = value[0:1]
# for i in xrange(1, len(value)):
# if start[0] != value[i]:
# break
# start += value[i]
#
# value = value[len(start):]
# start = escape_tex(start, self.commandprefix)
#
# # ... but do not escape inside comment.
# value = start + value
# elif self.mathescape:
# # Only escape parts not inside a math environment.
# parts = value.split('$')
# in_math = False
# for i, part in enumerate(parts):
# if not in_math:
# parts[i] = escape_tex(part, self.commandprefix)
# in_math = not in_math
# value = '$'.join(parts)
# else:
# value = escape_tex(value, self.commandprefix)
# else:
# value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("(*@\\%s{%s}{%s}@*)" %
(cp, styleval, escape_tex(line)))
outfile.write('\n')
if spl[-1]:
outfile.write("(*@\\%s{%s}{%s}@*)" %
(cp, styleval, escape_tex(spl[-1])))
else:
outfile.write(value)
outfile.write('\\end{lstlisting}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass=self.docclass,
preamble=self.preamble,
title=self.title,
encoding=self.encoding or 'utf-8',
styledefs=self.get_style_defs(),
code=outfile.getvalue()))
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(u'\\begin{' + self.envname +
u'}[commandchars=\\\\\\{\\}')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(u',numbers=left' +
(start and u',firstnumber=%d' % start or u'') +
(step and u',stepnumber=%d' % step or u''))
if self.mathescape or self.texcomments or self.escapeinside:
outfile.write(
u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
if self.verboptions:
outfile.write(u',' + self.verboptions)
outfile.write(u']\n')
for ttype, value in tokensource:
if ttype in Token.Comment:
if self.texcomments:
# Try to guess comment starting lexeme and escape it ...
start = value[0:1]
for i in xrange(1, len(value)):
if start[0] != value[i]:
break
start += value[i]
value = value[len(start):]
start = escape_tex(start, cp)
# ... but do not escape inside comment.
value = start + value
elif self.mathescape:
# Only escape parts not inside a math environment.
parts = value.split('$')
in_math = False
for i, part in enumerate(parts):
if not in_math:
parts[i] = escape_tex(part, cp)
in_math = not in_math
value = '$'.join(parts)
elif self.escapeinside:
text = value
value = ''
while text:
a, sep1, text = text.partition(self.left)
if sep1:
b, sep2, text = text.partition(self.right)
if sep2:
value += escape_tex(a, cp) + b
else:
value += escape_tex(a + sep1 + b, cp)
else:
value += escape_tex(a, cp)
else:
value = escape_tex(value, cp)
elif ttype not in Token.Escape:
value = escape_tex(value, cp)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write(u'\\end{' + self.envname + u'}\n')
if self.full:
encoding = self.encoding or 'utf8'
# map known existings encodings from LaTeX distribution
encoding = {
'utf_8': 'utf8',
'latin_1': 'latin1',
'iso_8859_1': 'latin1',
}.get(encoding.replace('-', '_'), encoding)
realoutfile.write(DOC_TEMPLATE %
dict(docclass=self.docclass,
preamble=self.preamble,
title=self.title,
encoding=encoding,
styledefs=self.get_style_defs(),
code=outfile.getvalue()))
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(r'\begin{Verbatim}[commandchars=@\[\]')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(',numbers=left' +
(start and ',firstnumber=%d' % start or '') +
(step and ',stepnumber=%d' % step or ''))
if self.verboptions:
outfile.write(',' + self.verboptions)
outfile.write(']\n')
for ttype, value in tokensource:
value = escape_tex(value, self.commandprefix)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("@%s[%s][%s]" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("@%s[%s][%s]" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write('\\end{Verbatim}\n')
if self.full:
realoutfile.write(DOC_TEMPLATE %
dict(docclass=self.docclass,
preamble=self.preamble,
title=self.title,
encoding=self.encoding or 'latin1',
styledefs=self.get_style_defs(),
code=outfile.getvalue()))
def check(optdict):
outfile = StringIO()
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
def _wrap_tablelinenos(self, inner):
dummyoutfile = StringIO()
lncount = 0
for t, line in inner:
if t:
lncount += 1
dummyoutfile.write(line)
fl = self.linenostart # 行号起始
mw = len(str(lncount + fl - 1)) # 最大行号宽度
sp = self.linenospecial # 特殊行间隔数
st = self.linenostep # 行号间隔数
la = self.lineanchors # 行锚点名称
aln = self.anchorlinenos # 如果设置为True，会给行号添加一个a标签，和`linenos` and `lineanchors`联合使用
nocls = self.noclasses # 是否是无class模式
if sp:
lines = []
for i in range(fl, fl + lncount):
if i % st == 0:
if i % sp == 0:
# 特殊行号
if aln:
# 锚点
lines.append(
'<a href="#%s-%d" class="special">%*d</a>' %
(la, i, mw, i))
else:
lines.append('<span class="special">%*d</span>' %
(mw, i))
else:
if aln:
# 锚点
lines.append('<a href="#%s-%d">%*d</a>' %
(la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
else:
lines = []
for i in range(fl, fl + lncount):
if i % st == 0:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' %
(la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
# in case you wonder about the seemingly redundant <div> here: since the
# content in the other cell also is wrapped in a div, some browsers in
# some configurations seem to mess up the formatting...
style = []
if nocls:
style.append('background-color: #f0f0f0')
style.append('padding-right: 10px')
clazz = []
if self.cssclass:
clazz.append('%stable' % self.cssclass)
# 自动换行
if self.linefeed:
if not self.nocls:
clazz.append('linefeed')
clazz = ' '.join(clazz)
style = '; '.join(style)
if nocls:
yield 0, ('<table class="%s">' % clazz +
'<tr><td><div class="linenodiv" '
'style="%s">' + style +
'<pre style="line-height: 125%">' + ls +
'</pre></div></td><td class="code">')
else:
yield 0, ('<table class="%s">' % clazz +
'<tr><td class="linenos"><div class="linenodiv"><pre>' +
ls + '</pre></div></td><td class="code">')
yield 0, dummyoutfile.getvalue()
yield 0, '</td></tr></table>'