def render(self, results):
target = None
self.trunc_top = self.args.reverse
t_res, m_res = results
if t_res and t_res.is_success and len(t_res.targets) > 0:
target = t_res.targets[0]
if self.args.deref or self.args.words:
self.args.bytes = target['addr_size']
f = pygments.formatters.get_formatter_by_name(self.config.format.pygments_formatter,
style=self.config.format.pygments_style)
if m_res and m_res.is_success:
lines = pygments.format(self.generate_tokens(results), f).split('\n')
self.body = '\n'.join(reversed(lines)).strip() if self.args.reverse else '\n'.join(lines)
self.info = '[0x{0:0=4x}:'.format(len(m_res.memory)) + self.config.format.addr_format.format(m_res.address) + ']'
else:
log.error("Error reading memory: {}".format(m_res.message))
self.body = pygments.format([(Error, m_res.message)], f)
self.info = ''
# Store the memory
if self.args.track:
self.last_address = m_res.address
self.last_memory = m_res.memory
else:
self.body = self.colour("Failed to get targets", 'red')
if not self.title:
self.title = "[memory]"
super(MemoryView, self).render(results)
def test_unicode_handling(self):
# test that the formatter supports encoding and Unicode
tokens = list(lexers.PythonLexer(encoding='utf-8').
get_tokens("def f(): 'ä'"))
for formatter, info in formatters.FORMATTERS.iteritems():
try:
inst = formatter(encoding=None)
except (ImportError, FontNotFound):
# some dependency or font not installed
continue
if formatter.name != 'Raw tokens':
out = format(tokens, inst)
if formatter.unicodeoutput:
self.assert_(type(out) is unicode)
inst = formatter(encoding='utf-8')
out = format(tokens, inst)
self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))
# Cannot test for encoding, since formatters may have to escape
# non-ASCII characters.
else:
inst = formatter()
out = format(tokens, inst)
self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))
def format(tok, tik=None):
if tik:
tok = (tok, tik)
if isinstance(tok, tuple):
return pygments.format([tok], formatter)
else:
return pygments.format(tok, formatter)
def test_bare_class_handler():
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer
try:
lex('test\n', PythonLexer)
except TypeError as e:
assert 'lex() argument must be a lexer instance' in str(e)
else:
assert False, 'nothing raised'
try:
format([], HtmlFormatter)
except TypeError as e:
assert 'format() argument must be a formatter instance' in str(e)
else:
assert False, 'nothing raised'
def test_unicode_handling(self):
# test that the formatter supports encoding and Unicode
tokens = list(lexers.PythonLexer(encoding='utf-8').get_tokens("def f(): 'ä'"))
for formatter, info in formatters.FORMATTERS.iteritems():
try:
inst = formatter(encoding=None)
except (ImportError, FontNotFound):
# some dependency or font not installed
continue
out = format(tokens, inst)
if formatter.unicodeoutput:
self.assert_(type(out) is unicode)
inst = formatter(encoding='utf-8')
out = format(tokens, inst)
self.assert_(type(out) is str)
def push(self, line):
"""Push a line of code onto the buffer, run the buffer
If the interpreter successfully runs the code, clear the buffer
Return ("for stdout", "for_stderr", finished?)
"""
self.buffer.append(line)
indent = len(re.match(r'[ ]*', line).group())
if line.endswith(':'):
indent = max(0, indent + INDENT_AMOUNT)
elif line and line.count(' ') == len(self._current_line):
indent = max(0, indent - INDENT_AMOUNT)
elif line and ':' not in line and line.strip().startswith(('return', 'pass', 'raise', 'yield')):
indent = max(0, indent - INDENT_AMOUNT)
out_spot = sys.stdout.tell()
err_spot = sys.stderr.tell()
#logging.debug('running %r in interpreter', self.buffer)
unfinished = self.interp.runsource('\n'.join(self.buffer))
self.display_buffer.append(bpythonparse(format(self.tokenize(line), self.formatter))) #current line not added to display buffer if quitting
sys.stdout.seek(out_spot)
sys.stderr.seek(err_spot)
out = sys.stdout.read()
err = sys.stderr.read()
if unfinished and not err:
logging.debug('unfinished - line added to buffer')
return (None, None, False, indent)
else:
logging.debug('finished - buffer cleared')
self.display_lines.extend(self.display_buffer_lines)
self.display_buffer = []
self.buffer = []
if err:
indent = 0
return (out[:-1], err[:-1], True, indent)
def string_to_fmtstr(x):
from pygments import format
from bpython.formatter import BPythonFormatter
from bpython._py3compat import PythonLexer
from bpython.config import Struct, loadini, default_config_path
config = Struct()
loadini(config, default_config_path())
return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))
def highlightBlock(self, text): """Takes a block, applies format to the document. according to what's in it. """ # I need to know where in the document we are, # because our formatting info is global to # the document cb = self.currentBlock() p = cb.position() '''print cb print p print cb.text()''' blockText = unicode(cb.text())+'\n' # The \n is not really needed, but sometimes # you are in an empty last block, so your position is # **after** the end of the document. text=unicode(self.document().toPlainText())+'\n' # Yes, re-highlight the whole document. # There **must** be some optimization possibilities # but it seems fast enough. #highlight(blockText,self.lexer,self.formatter) tokens = pygments.lex(blockText, self.lexer) self.docTokens[cb.blockNumber()] = tokens pygments.format(tokens, self.formatter) data = self.formatter.getData() # Just apply the formatting to this block. # For titles, it may be necessary to backtrack # and format a couple of blocks **earlier**. for i in range(len(unicode(blockText))): try: self.setFormat(i,1,data[i]) except IndexError: pass # I may need to do something about this being called # too quickly. self.tstamp=time.time()
def format(cls, tokens):
"""Format the given list of tokens
:param tokens: the input list of token to format
:type tokens: tuple[Token, str]
:rtype: str
"""
cls.__initialize()
return pygments.format(tokens, cls.formatter)
def test_formatter_encodings():
from pygments.formatters import HtmlFormatter
# unicode output
fmt = HtmlFormatter()
tokens = [(Text, u"ä")]
out = format(tokens, fmt)
assert type(out) is text_type
assert u"ä" in out
# encoding option
fmt = HtmlFormatter(encoding="latin1")
tokens = [(Text, u"ä")]
assert u"ä".encode("latin1") in format(tokens, fmt)
# encoding and outencoding option
fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
tokens = [(Text, u"ä")]
assert u"ä".encode("utf8") in format(tokens, fmt)
def test_encodings(self):
from pygments.formatters import HtmlFormatter
# unicode output
fmt = HtmlFormatter()
tokens = [(Text, u"ä")]
out = format(tokens, fmt)
self.assert_(type(out) is unicode)
self.assert_(u"ä" in out)
# encoding option
fmt = HtmlFormatter(encoding="latin1")
tokens = [(Text, u"ä")]
self.assert_(u"ä".encode("latin1") in format(tokens, fmt))
# encoding and outencoding option
fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
tokens = [(Text, u"ä")]
self.assert_(u"ä".encode("utf8") in format(tokens, fmt))
def test_formatter_encodings():
from pygments.formatters import HtmlFormatter
# unicode output
fmt = HtmlFormatter()
tokens = [(Text, u"ä")]
out = format(tokens, fmt)
assert type(out) is unicode
assert u"ä" in out
# encoding option
fmt = HtmlFormatter(encoding="latin1")
tokens = [(Text, u"ä")]
assert u"ä".encode("latin1") in format(tokens, fmt)
# encoding and outencoding option
fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
tokens = [(Text, u"ä")]
assert u"ä".encode("utf8") in format(tokens, fmt)
def raw_format(cls, tokens):
"""Format the given list of tokens as a simple string (no color)
:param tokens: the input list of token to format
:type tokens: tuple[Token, str]
:rtype: str
"""
formatter = get_formatter_by_name(Formatter.NO_COLOR, encoding=Formatter.ENCODING)
return pygments.format(tokens, formatter)
def print_color(self, string, hide=False, **kwargs):
if isinstance(string, str):
s = self.format_color(string, hide=hide)
else:
# assume this is a list of (Token, str) tuples and format it
env = builtins.__xonsh_env__
self.styler.style_name = env.get('XONSH_COLOR_STYLE')
style_proxy = pyghooks.xonsh_style_proxy(self.styler)
formatter = Terminal256Formatter(style=style_proxy)
s = pygments.format(string, formatter).rstrip()
print(s, **kwargs)
def print_color(self, string, hide=False, **kwargs):
if isinstance(string, str):
s = self.format_color(string, hide=hide)
else:
# assume this is a list of (Token, str) tuples and format it
env = builtins.__xonsh_env__
self.styler.style_name = env.get('XONSH_COLOR_STYLE')
style_proxy = pyghooks.xonsh_style_proxy(self.styler)
formatter = Terminal256Formatter(style=style_proxy)
s = pygments.format(string, formatter).rstrip()
print(s, **kwargs)
def raw_format(cls, tokens):
"""Format the given list of tokens as a simple string (no color)
:param tokens: the input list of token to format
:type tokens: tuple[Token, str]
:rtype: str
"""
formatter = get_formatter_by_name(Formatter.NO_COLOR,
encoding=Formatter.ENCODING)
return pygments.format(tokens, formatter)
def terminal(self):
"""
Return a Terminal-friendly (with ANSI color sequences) representation of the snippet.
"""
formatter = TerminalFormatter(
linenos=True,
colorscheme=None,
linenostart=self._start_line,
)
return pygments.format(self.src_tokens(), formatter)
def html(self):
"""
Return an HTML representation of the snippet.
"""
formatter = HtmlFormatter(cssclass=self.DIV_CSS_CLASS,
linenos=True,
linenostart=self._start_line,
hl_lines=self._shift_lines(
self._violation_lines, self._start_line),
lineanchors=self._src_filename)
return pygments.format(self.src_tokens(), formatter)
def current_line_formatted(self):
"""The colored current line (no prompt, not wrapped)"""
if self.config.syntax:
fs = bpythonparse(format(self.tokenize(self._current_line), self.formatter))
logging.debug('Display line %r -> %r', self._current_line, fs)
else:
fs = fmtstr(self._current_line)
if hasattr(self, 'old_fs') and str(fs) != str(self.old_fs):
pass
#logging.debug('calculating current formatted line: %r', repr(fs))
self.old_fs = fs
return fs
def extract_algorithms(ps_infile, env):
"""
Вытаскиваем части кода-алгоритмы
"""
import pygments
from pygments.lexers import get_lexer_by_name
from pygments.formatters import LatexFormatter
algorithm_regexp = re.compile(
r"(?ms)\#ALGORITHM\s+(?P<name>[a-zA-z-0-9]+)\s*(?P<code>.+?)\s*\#ENDALGORITHM")
hideline_regexps = [re.compile(r"(?m)^.*\#hide *\n"), re.compile(r"(?m)\n.*\#hide *") ]
ls = ut.file2string(ps_infile)
for algorithm in algorithm_regexp.finditer(ls):
algfilename = lib.get_target(ps_infile, algorithm.group('name')+".py")
texfilename = lib.get_target(ps_infile, algorithm.group('name')+".tex")
#lf = open(algfilename, 'w')
code = algorithm.group('code')
for r in hideline_regexps:
code = re.sub(r, "", code)
#code = lib.utf8anyway(code)
#lf.write(code)
#lf.close()
#tempblock = os.path.join(tempfile.gettempdir(), tempfile.gettempprefix())
#ls = ''.join([env.project_db['paths']['python'],
# r'\scripts\pygmentize -f latex -l python ',
# ' -o "%(tempblock)s" "%(algfilename)s" ' % vars() ])
#os.system(ls)
lexer = get_lexer_by_name('python')
code = ut.unicodeanyway(code)
latex_tokens = pygments.lex(code, lexer)
# sblock = ut.file2string(tempblock)
# from pygments.formatters import LatexFormatter
latex_formatter = LatexFormatter(texcomments = True)
latex = pygments.format(latex_tokens, latex_formatter)
stexblock = r"""
\documentclass{minimal}
\usepackage{xecyr}
\XeTeXdefaultencoding "utf-8"
\XeTeXinputencoding "utf-8"
\defaultfontfeatures{Mapping=tex-text}
\setmonofont{Consolas}
\usepackage{color}
\usepackage{fancyvrb}
\usepackage[russian,english]{babel}
""" + latex_formatter.get_style_defs() + r"""
\begin{document}
""" + latex + r"""
\end{document}
"""
ut.string2file(stexblock, texfilename, encoding='utf-8')
def verify(formatter):
try:
inst = formatter(encoding=None)
except (ImportError, FontNotFound):
# some dependency or font not installed
raise support.SkipTest
if formatter.name != 'Raw tokens':
out = format(tokens, inst)
if formatter.unicodeoutput:
assert type(out) is text_type, '%s: %r' % (formatter, out)
inst = formatter(encoding='utf-8')
out = format(tokens, inst)
assert type(out) is bytes, '%s: %r' % (formatter, out)
# Cannot test for encoding, since formatters may have to escape
# non-ASCII characters.
else:
inst = formatter()
out = format(tokens, inst)
assert type(out) is bytes, '%s: %r' % (formatter, out)
def verify(formatter):
try:
inst = formatter(encoding=None)
except (ImportError, FontNotFound):
# some dependency or font not installed
raise support.SkipTest
if formatter.name != 'Raw tokens':
out = format(tokens, inst)
if formatter.unicodeoutput:
assert type(out) is text_type, '%s: %r' % (formatter, out)
inst = formatter(encoding='utf-8')
out = format(tokens, inst)
assert type(out) is bytes, '%s: %r' % (formatter, out)
# Cannot test for encoding, since formatters may have to escape
# non-ASCII characters.
else:
inst = formatter()
out = format(tokens, inst)
assert type(out) is bytes, '%s: %r' % (formatter, out)
def verify(formatter):
try:
inst = formatter(encoding=None)
except (ImportError, FontNotFound):
# some dependency or font not installed
return
if formatter.name != "Raw tokens":
out = format(tokens, inst)
if formatter.unicodeoutput:
assert type(out) is unicode
inst = formatter(encoding="utf-8")
out = format(tokens, inst)
assert type(out) is bytes, "%s: %r" % (formatter, out)
# Cannot test for encoding, since formatters may have to escape
# non-ASCII characters.
else:
inst = formatter()
out = format(tokens, inst)
assert type(out) is bytes, "%s: %r" % (formatter, out)
def log(self, tokens, stream=sys.stdout):
"""Log the input token stream with the standard Python logging
mecanism.
PARAMETERS
log_fn: the logging function to use
tokens: the input tokens stream
"""
assert self.formatter is not None, 'Internal error'
print >> stream, pygments.format(tokens, self.formatter)
stream.flush()
def format_wkt_for_output(output, fp=None, syntax_highlight=True):
"""
Formats WKT whitespace for readability.
Adds syntax highlighting if fp is a terminal and syntax_highlight=True.
Doesn't print the formatted WKT to fp, just returns it.
"""
token_iter = WKTLexer().get_tokens(output, pretty_print=True)
if syntax_highlight and can_output_colour(fp):
return pygments.format(token_iter, get_terminal_formatter())
else:
token_value = (value for token_type, value in token_iter)
return "".join(token_value)
def unhighlight_paren(self):
"""set self.display_buffer after """
if self.highlighted_paren is not None:
lineno, saved_tokens = self.highlighted_paren
if lineno == len(self.display_buffer):
# then this is the current line, so don't worry about it
return
self.highlighted_paren = None
logging.debug('trying to unhighlight a paren on line %r', lineno)
logging.debug('with these tokens: %r', saved_tokens)
new = bpythonparse(format(saved_tokens, self.formatter))
self.display_buffer[lineno][:len(new)] = new
def print_result(self,
result,
prompt: bool,
output_style="",
strict_wl_output=False):
if result is None:
# FIXME decide what to do here
return
last_eval = result.last_eval
if last_eval is not None:
try:
eval_type = last_eval.get_head_name()
except:
print(sys.exc_info()[1])
return
out_str = str(result.result)
use_highlight = True
if eval_type == "System`String":
if strict_wl_output: # exact-wl-compatibility
out_str = (format([(MToken.STRING, out_str.rstrip())],
self.terminal_formatter) + "\n")
use_highlight = False
else:
out_str = '"' + out_str.replace('"', r"\"") + '"'
if eval_type == "System`Graph":
out_str = "-Graph-"
elif self.terminal_formatter: # pygmentize
show_pygments_tokens = self.definitions.get_ownvalue(
"Settings`$PygmentsShowTokens").replace.to_python()
pygments_style = self.definitions.get_ownvalue(
"Settings`$PygmentsStyle").replace.get_string_value()
if pygments_style != self.pygments_style:
if not self.change_pygments_style(pygments_style):
self.definitions.set_ownvalue(
"Settings`$PygmentsStyle",
String(self.pygments_style))
if show_pygments_tokens:
print(list(lex(out_str, mma_lexer)))
if use_highlight:
out_str = highlight(out_str, mma_lexer,
self.terminal_formatter)
output = self.to_output(out_str)
if output_style == "text" or not prompt:
print(output)
elif self.session:
print_formatted_text(self.get_out_prompt(), end="")
print(output + "\n")
else:
print(self.get_out_prompt() + output + "\n")
def render(self, results):
target = None
self.trunc_top = self.args.reverse
t_res, m_res = results
if t_res and t_res.is_success and len(t_res.targets) > 0:
target = t_res.targets[0]
if self.args.deref or self.args.words:
self.args.bytes = target['addr_size']
f = pygments.formatters.get_formatter_by_name(
self.config.format.pygments_formatter,
style=self.config.format.pygments_style)
if m_res and m_res.is_success:
lines = pygments.format(self.generate_tokens(results),
f).split('\n')
self.body = '\n'.join(reversed(
lines)).strip() if self.args.reverse else '\n'.join(lines)
self.info = '[0x{0:0=4x}:'.format(len(
m_res.memory)) + self.config.format.addr_format.format(
m_res.address) + ']'
else:
log.error("Error reading memory: {}".format(m_res.message))
self.body = pygments.format([(Error, m_res.message)], f)
self.info = ''
# Store the memory
if self.args.track:
self.last_address = m_res.address
self.last_memory = m_res.memory
else:
self.body = self.colour("Failed to get targets", 'red')
if not self.title:
self.title = "[memory]"
super(MemoryView, self).render(results)
def insert_formatted_code(tb,
language,
code,
mark=None,
line_break=False):
lexer = None
if language is None:
log.info(
"No Language specified. Falling back to default lexer: %s.",
str(self.config['default_lexer']))
lexer = get_lexer(self.config['default_lexer'])
else:
log.debug("Using lexer for %s.", str(language))
lexer = get_lexer_with_fallback(language,
self.config['default_lexer'])
if lexer is None:
it = tb.get_iter_at_mark(mark)
tb.insert(it, '\n')
else:
tokens = pygments.lex(code, lexer)
if line_break:
log.debug("Inserting newline before code.")
it = tb.get_iter_at_mark(mark)
tb.insert(it, '\n')
it.forward_char()
tb.move_mark(mark, it)
formatter = GTKFormatter(start_mark=mark)
pygments.format(tokens, formatter, tb)
endmark = formatter.get_last_mark()
if line_break and not endmark is None:
it = tb.get_iter_at_mark(endmark)
tb.insert(it, '\n')
log.debug("Inserting newline after code.")
return tb
def test_bare_class_handler():
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer
try:
lex('test\n', PythonLexer)
except TypeError as e:
assert 'lex() argument must be a lexer instance' in str(e)
else:
assert False, 'nothing raised'
try:
format([], HtmlFormatter)
except TypeError as e:
assert 'format() argument must be a formatter instance' in str(e)
else:
assert False, 'nothing raised'
# These cases should not trigger this heuristic.
class BuggyLexer(RegexLexer):
def get_tokens(self, text, extra_argument):
pass
tokens = {'root': []}
try:
list(lex('dummy', BuggyLexer()))
except TypeError as e:
assert 'lex() argument must be a lexer instance' not in str(e)
else:
assert False, 'no error raised by buggy lexer?'
class BuggyFormatter(Formatter):
def format(self, tokensource, outfile, extra_argument):
pass
try:
format([], BuggyFormatter())
except TypeError as e:
assert 'format() argument must be a formatter instance' not in str(e)
else:
assert False, 'no error raised by buggy formatter?'
def write_module_coverage_file(self, app, moduleName, sourceFileName, num_of_lines, not_covered_lines = [], excluded_lines = []):
''' Set and writes the coverage report '''
from django.template import Context
from django.template.loader import get_template
#Decode a file
fo = codecs.open(sourceFileName, 'rb', "utf-8")
try:
source = fo.read()
finally:
fo.close()
try:
offset = 0
lines = source.split("\n")
while lines[ offset ] == "":
offset += 1
if offset > 0:
not_covered_lines = [x - 1 for x in not_covered_lines]
except IndexError:
offset = 0
#Lexer tokenize an input string
lexer = get_lexer_by_name("py")
tokens = lex(source, lexer)
fmt = CssHtmlFormatter(linenos = 'inline', hl_lines = not_covered_lines, noclasses = False, css_lines = {"skipped" : excluded_lines})
fmt.lineseparator = "\n"
source_html = format(tokens, fmt)
ncl = len(not_covered_lines) # uncovered lines
cl = num_of_lines - ncl # number covered lines
el = len(excluded_lines)
co = cl > 0 and float(cl * 100 / num_of_lines) or 0
t = get_template('coverage_module.tpl.html')
now = datetime.datetime.now()
html = t.render(Context({'now': now,
'module':moduleName,
'app':moduleName.split('.')[0],
'pkgs':moduleName.split("."),
'tl': num_of_lines,
'cl': cl,
'el': el,
'co': co,
'title': "%s coverage" % moduleName,
'code': source_html,
'GLOBAL_COVERAGE': GLOBAL_COVERAGE,
}))
fp = self.get_file_handle(app, moduleName)
fp.write(html.encode('utf-8'))
fp.close()
def format(cls, tokens):
"""
Format the given list of tokens.
PARAMETERS
tokens: the input list of token to format
RETURNS
the formatted string
"""
cls.__initialize()
return pygments.format(tokens, cls.formatter)
def html(self):
"""
Return an HTML representation of the snippet.
"""
formatter = HtmlFormatter(
cssclass=self.DIV_CSS_CLASS,
linenos=True,
linenostart=self._start_line,
hl_lines=self._shift_lines(self._violation_lines, self._start_line),
lineanchors=self._src_filename,
)
return pygments.format(self.src_tokens(), formatter)
def format_wkt_for_output(output, fp):
"""
If the given filelike object is a terminal, adds whitespace and syntax highlighting to the output.
Doesn't actually write the output, just returns it.
"""
tokens_with_whitespace = wkt_whitespace_format(
WKTLexer().get_tokens(output))
if fp == sys.stdout and fp.isatty():
return pygments.format(tokens_with_whitespace,
get_terminal_formatter())
else:
token_value = (value for token_type, value in tokens_with_whitespace)
return "".join(token_value)
def format_code(self, buf, s_tag, s_code, e_tag, e_code, language):
style = self.config.get_style_name()
if self.config.get_code_marker_setting() == CodeMarkerOptions.HIDE:
self.hide_code_markup(buf, s_tag, s_code)
self.hide_code_markup(buf, e_code, e_tag)
else:
comment_tag = GTKFormatter.create_tag_for_token(
pygments.token.Comment,
pygments.styles.get_style_by_name(style))
buf.get_tag_table().add(comment_tag)
buf.apply_tag(comment_tag, s_tag, s_code)
buf.apply_tag(comment_tag, e_tag, e_code)
code = s_code.get_text(e_code)
log.debug('full text to encode: %s.', code)
start_mark = buf.create_mark(None, s_code, False)
lexer = None
if language is None:
lexer = self.config.get_default_lexer()
log.info(
'No Language specified. '
'Falling back to default lexer: %s.',
self.config.get_default_lexer_name())
else:
log.debug('Using lexer for %s.', str(language))
lexer = self.config.get_lexer_with_fallback(language)
if lexer is None:
iterator = buf.get_iter_at_mark(start_mark)
buf.insert(iterator, '\n')
elif not self.config.is_internal_none_lexer(lexer):
tokens = pygments.lex(code, lexer)
formatter = GTKFormatter(style=style, start_mark=start_mark)
pygments.format(tokens, formatter, buf)
def raw_format(cls, tokens):
"""
Format the given list of tokens as a simple string (no color).
PARAMETERS
tokens: the input list of token to format
RETURNS
the formatted string
"""
formatter = get_formatter_by_name(Formatter.NO_COLOR,
encoding=Formatter.ENCODING)
return pygments.format(tokens, formatter)
def _format_preview_text(self):
buf = self._ui.preview_textview.get_buffer()
start_iter = buf.get_start_iter()
start_mark = buf.create_mark(None, start_iter, True)
buf.remove_all_tags(start_iter, buf.get_end_iter())
formatter = GTKFormatter(style=self.config.get_style_name(),
start_mark=start_mark)
code = start_iter.get_text(buf.get_end_iter())
lexer = self.config.get_default_lexer()
if not self.config.is_internal_none_lexer(lexer):
tokens = pygments.lex(code, lexer)
pygments.format(tokens, formatter, buf)
buf.delete_mark(start_mark)
self._ui.preview_textview.override_font(
FontDescription.from_string(self.config.get_font()))
color = Gdk.RGBA()
if color.parse(self.config.get_bgcolor()):
self._ui.preview_textview.override_background_color(
Gtk.StateFlags.NORMAL, color)
def format_color(self, string, hide=False, force_string=False, **kwargs):
"""Formats a color string using Pygments. This, therefore, returns
a list of (Token, str) tuples. If force_string is set to true, though,
this will return a color fomtatted string.
"""
tokens = partial_color_tokenize(string)
if force_string:
env = builtins.__xonsh_env__
self.styler.style_name = env.get('XONSH_COLOR_STYLE')
proxy_style = xonsh_style_proxy(self.styler)
formatter = XonshTerminal256Formatter(style=proxy_style)
s = pygments.format(tokens, formatter)
return s
else:
return tokens
def fill_read_cache(self, filename: str, line: int) -> None:
try:
lexer = pygments.lexers.get_lexer_for_filename(
str(filename)) # type: RegexLexer
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs(".highlight")
with open(str(filename)) as f:
lines = f.readlines()
if len(lines) < 1000:
content = "".join(lines)
tokens = lexer.get_tokens(content)
source = pygments.format(tokens, html_formatter)
self.file_cache[filename][line] = (css, source)
self.file_read_cache[filename] = (lexer, content, False)
else:
minl = max(0, line - 30)
maxl = min(len(lines), line + 30)
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs(".highlight")
source = pygments.format(
lexer.get_tokens("".join(lines[minl:maxl])),
html_formatter)
self.file_cache[filename][line] = (css, source)
self.file_read_cache[filename] = (lexer, lines, True)
except Exception as e:
l.exception(e)
self.file_cache[filename][line] = (None, None)
self.file_read_cache[filename] = (None, None, False)
def unhighlight_paren(self):
"""modify line in self.display_buffer to unhighlight a paren if possible
self.highlighted_paren should be a line in ?
"""
if self.highlighted_paren is not None and self.config.syntax:
lineno, saved_tokens = self.highlighted_paren
if lineno == len(self.display_buffer):
# then this is the current line, so don't worry about it
return
self.highlighted_paren = None
logging.debug('trying to unhighlight a paren on line %r', lineno)
logging.debug('with these tokens: %r', saved_tokens)
new = bpythonparse(format(saved_tokens, self.formatter))
self.display_buffer[lineno] = self.display_buffer[lineno].setslice_with_length(0, len(new), new, len(self.display_buffer[lineno]))
def format_color(self, string, hide=False, force_string=False, **kwargs):
"""Formats a color string using Pygments. This, therefore, returns
a list of (Token, str) tuples. If force_string is set to true, though,
this will return a color fomtatted string.
"""
tokens = partial_color_tokenize(string)
if force_string:
env = builtins.__xonsh_env__
self.styler.style_name = env.get('XONSH_COLOR_STYLE')
proxy_style = xonsh_style_proxy(self.styler)
formatter = XonshTerminal256Formatter(style=proxy_style)
s = pygments.format(tokens, formatter)
return s
else:
return tokens
def push(self, line, insert_into_history=True):
"""Push a line of code onto the buffer, start running the buffer
If the interpreter successfully runs the code, clear the buffer
"""
if self.paste_mode:
self.saved_indent = 0
else:
indent = len(re.match(r'[ ]*', line).group())
if line.endswith(':'):
indent = max(0, indent + self.config.tab_length)
elif line and line.count(' ') == len(line):
indent = max(0, indent - self.config.tab_length)
elif line and ':' not in line and line.strip().startswith(('return', 'pass', 'raise', 'yield')):
indent = max(0, indent - self.config.tab_length)
self.saved_indent = indent
#current line not added to display buffer if quitting #TODO I don't understand this comment
if self.config.syntax:
display_line = bpythonparse(format(self.tokenize(line), self.formatter))
# careful: self.tokenize requires that the line not be in self.buffer yet!
logging.debug('display line being pushed to buffer: %r -> %r', line, display_line)
self.display_buffer.append(display_line)
else:
self.display_buffer.append(fmtstr(line))
if insert_into_history:
self.insert_into_history(line)
self.buffer.append(line)
code_to_run = '\n'.join(self.buffer)
logging.debug('running %r in interpreter', self.buffer)
try:
c = bool(code.compile_command('\n'.join(self.buffer)))
self.saved_predicted_parse_error = False
except (ValueError, SyntaxError, OverflowError):
c = self.saved_predicted_parse_error = True
if c:
logging.debug('finished - buffer cleared')
self.display_lines.extend(self.display_buffer_lines)
self.display_buffer = []
self.buffer = []
self.cursor_offset_in_line = 0
self.coderunner.load_code(code_to_run)
self.run_code_and_maybe_finish()
def test():
from pygments import format
from bpython.formatter import BPythonFormatter
from bpython._py3compat import PythonLexer
from bpython.config import Struct, loadini, default_config_path
config = Struct()
loadini(config, default_config_path())
all_tokens = list(PythonLexer().get_tokens('print(1 + 2)'))
formatted_line = format(all_tokens, BPythonFormatter(config.color_scheme))
print((repr(formatted_line)))
fs = parse(formatted_line)
print((repr(fs)))
print(fs)
string_to_fmtstr('asdf')
def visit_code_block(self,node):
inline = isinstance(node.parent, nodes.TextElement)
attrs = node.attributes
if inline:
self.body.append('\\lstinline{%s}' % node.latex)
else:
if node.formatter == 'pygments':
lexer = get_lexer_by_name(language)
latex_tokens = pygments.lex(code, lexer)
formatter = LatexFormatter()
latex = [pygments.format(latex_tokens,formatter)]
elif node.formatter == 'listings':
latex = ['\\begin{lstlisting}[language=%s]\n'%node.language,
node.latex,
'\n\\end{lstlisting}\n']
self.body.extend(latex)
def test():
from pygments import format
from bpython.formatter import BPythonFormatter
from bpython._py3compat import PythonLexer
from bpython.config import Struct, loadini, default_config_path
config = Struct()
loadini(config, default_config_path())
all_tokens = list(PythonLexer().get_tokens('print 1 + 2'))
formatted_line = format(all_tokens, BPythonFormatter(config.color_scheme))
print((repr(formatted_line)))
fs = parse(formatted_line)
print((repr(fs)))
print(fs)
string_to_fmtstr('asdf')
def viewParameterInSimulation(request, species_wid, wid):
#get associated simulation property
qs = Parameter.objects.filter(species__wid = species_wid, wid=wid)
if not qs[0].state is None:
sim_class_name = 'edu.stanford.covert.cell.sim.state.%s' % qs[0].state.wid.replace('State_', '')
verbose_class_name = '%s: %s' % (wid, qs[0].state.name)
else:
sim_class_name = 'edu.stanford.covert.cell.sim.process.%s' % qs[0].process.wid.replace('Process_', '')
verbose_class_name = '%s: %s' % (wid, qs[0].process.name)
sim_property_name = qs[0].name
verbose_property_name = qs[0].name
#highlight code for simulation class
pathParts = sim_class_name.split('.')
codePath = "%s/src/+%s/%s.m" % (MODEL_CODE_BASE_DIR, '/+'.join(pathParts[0:-1]), pathParts[-1])
if not os.path.isfile(codePath):
codePath = "%s/src/+%s/@%s/%s.m" % (MODEL_CODE_BASE_DIR, '/+'.join(pathParts[0:-1]), pathParts[-1], pathParts[-1])
if os.path.isfile(codePath):
with open (codePath, "r") as codeFile:
code = codeFile.read()
lexer = MatlabLexer()
lexer.add_filter(PropertyDefinitionFilter(property_names = [sim_property_name], tokentype=Token.Name.Variable))
tokens = lexer.get_tokens(code)
object = {
'class_name': sim_class_name,
'property_names': [sim_property_name],
'code': pygments.format(tokens, PygmentsFormatter(linenos='inline', linenostep=1, style=PygmentsStyle, noclasses=True)),
}
else:
raise Http404
#render response
return render_queryset_to_response(
species_wid = species_wid,
request = request,
models = [Parameter],
queryset = qs,
templateFile = 'public/viewPropertyInSimulation.html',
data = {
'object_list': [object],
'verbose_class_name': verbose_class_name,
'verbose_property_name': verbose_property_name,
})
def run(self):
formatter = self.state.document.settings.code_block_formatter
if self.options.has_key('formatter'):
echo = self.options['formatter']
else:
formatter = 'listings'
language = self.arguments[0]
code = ''
for line in self.content:
code+='%s\n'%line
if formatter == 'pygments':
lexer = get_lexer_by_name(language)
latex_tokens = pygments.lex(code, lexer)
formatter = LatexFormatter()
latex = pygments.format(latex_tokens,formatter)
elif formatter == 'listings':
latex = code
node = code_block(self.block_text,latex,language,formatter)
return [node]
def format_code(x, xlate=True, to_try=None, no_errors=False, with_meta=False):
try:
import pygments, pygments.formatters, pygments.lexers
except Exception as e:
print "exception trying to format code: {0}".format(e)
return x
f = pygments.formatters.TerminalFormatter()
if to_try is None:
to_try = ('json', 'yaml', 'python')
if not isinstance(to_try, (tuple,list)):
to_try = (to_try,)
best_tokens = False
for lname in to_try:
lexer = pygments.lexers.get_lexer_by_name(lname)
tokens,spare_tokens = itertools.tee(pygments.lex(x, lexer))
c = 0
for t in spare_tokens:
if t[0][0] == 'Error':
c += 1
if no_errors and c:
continue
if best_tokens and c >= best_tokens[0]:
continue
best_tokens = (c, tokens, lname)
if best_tokens:
formatted = pygments.format(best_tokens[1],f)
if xlate:
formatted = xlate_ansi( formatted )
if with_meta:
return {'errors': best_tokens[0], 'formatted': formatted, 'lexer': best_tokens[2]}
return formatted
return x
def output_stage(name, function):
if 'dot' in args.dumps:
dot_document = function_to_dot(function, style=args.style)
if args.dot_format:
dot_args = ['dot',
'-T{}'.format(args.dot_format),
'-o' '{}.{}'.format(name, args.dot_format),
]
if args.dpi is not None:
dot_args.append('-Gdpi={}'.format(args.dpi))
dot = subprocess.Popen(dot_args, stdin=subprocess.PIPE)
dot.communicate(dot_document.encode('utf-8'))
else:
with open('{}.dot'.format(name), 'w') as f:
f.write(dot_document)
f.write('\n')
if 'text' in args.dumps:
with open('{}.ll'.format(name), 'w') as f:
f.write(pygments.format(function.format(), text_formatter))
f.write('\n')
def highlight(self, block):
"""Method called on each block to highlight it content"""
tokens = pygments.lex(block, self.python_lexer)
if self.format_rst:
from pygments.token import Token
toks = []
for token in tokens:
if token[0] == Token.String.Doc and len(token[1]) > 6:
toks += pygments.lex(token[1][:3], self.python_lexer)
# parse doc string content by rst lexer
toks += pygments.lex(token[1][3:-3], self.rst_lexer)
toks += pygments.lex(token[1][-3:], self.python_lexer)
elif token[0] == Token.Comment.Single:
toks.append((Token.Comment.Single, token[1][0]))
# parse comment content by rst lexer
# remove the extra newline added by rst lexer
toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1]
else:
toks.append(token)
tokens = toks
return pygments.format(tokens, self.formatter)
def output_stage(name, function):
if 'dot' in args.dumps:
dot_document = function_to_dot(function, style=args.style)
if args.dot_format:
dot_args = [
'dot',
'-T{}'.format(args.dot_format),
'-o'
'{}.{}'.format(name, args.dot_format),
]
if args.dpi is not None:
dot_args.append('-Gdpi={}'.format(args.dpi))
dot = subprocess.Popen(dot_args, stdin=subprocess.PIPE)
dot.communicate(dot_document.encode('utf-8'))
else:
with open('{}.dot'.format(name), 'w') as f:
f.write(dot_document)
f.write('\n')
if 'text' in args.dumps:
with open('{}.ll'.format(name), 'w') as f:
f.write(pygments.format(function.format(), text_formatter))
f.write('\n')
def highlight(self, lines):
pygments = self.module.pygments
tokens = pygments.lexers.PythonLexer().get_tokens("\n".join(lines))
source = pygments.format(tokens,
pygments.formatters.TerminalFormatter())
return source.split("\n")
def run_tests(test_files, index_file, index_file_base_path, amalgamation_file,
amalgamation_filename):
"""Run all tests and produce HTML render results.
Args:
test_files: An iterable of filenames to lex, test and render
index_file: A file object to write the HTML index to linking all render results
It must be opened as a text file with UTF-8 encoding.
index_file_base_path: Base path to use for links in index
amalgamation_file: A file object to write all HTML render results subsequently to
It must be opened as a binary file and it will be written to with UTF-8 encoding.
Return:
The number of files that failed complete lexing. On success, this is 0.
"""
lexer = MMTLexer(encoding="utf-8")
full_html_formatter = HtmlFormatter(full=True,
encoding="utf-8",
style=MMTDefaultStyle)
snippet_html_formatter = HtmlFormatter(full=False,
encoding="utf-8",
style=MMTDefaultStyle)
amalgamation_file.write(b"""
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<!-- Don't cache! -->
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate">
<meta http-equiv="Pragma" content="no-cache" />
<meta http-equiv="Expires" content="0">
<title>Amalgamation of Render Results - mmt-pygments-lexer</title>
</head>
<body>
<h1>Amalgamation of Render Results</h1>
""")
amalgamation_file.write(b"<style>")
amalgamation_file.write(
snippet_html_formatter.get_style_defs().encode("utf-8"))
amalgamation_file.write(b"</style>")
num_failing_files = 0
num_succeeding_lines = 0
out_statuses = []
# Tokens that we interpret as signalling a lexer error
# Token.Error is Pygment's standard error token whereas Generic.Error
# is issued by MMTLexer for graceful degradation
error_tokens = [Token.Error, Generic.Error]
for test_filename in test_files:
print("Running test for " + test_filename)
# We read both input and output file in binary mode to circumvent encoding issues
# Indeed, we specified above UTF-8 encoding for the lexer and formatter
with io.open(test_filename, mode="rb") as test_file:
tokens = list(lexer.get_tokens(test_file.read()))
erroneous = any(token in error_tokens for (token, _) in tokens)
if erroneous:
num_failing_files = num_failing_files + 1
print(
" --> Lexing error, see corresponding .html file for details\n"
)
else:
num_succeeding_lines = num_succeeding_lines + count_lines(
test_filename)
out_filename = test_filename + ".html"
out_statuses.append({"filename": out_filename, "error": erroneous})
with io.open(out_filename, mode="wb") as out_file:
pygments.format(tokens, full_html_formatter, out_file)
pygments.format(tokens, snippet_html_formatter, amalgamation_file)
print(" --> Output at " + out_filename)
generate_index_file(out_statuses, num_succeeding_lines, num_failing_files,
index_file_base_path, amalgamation_filename,
index_file)
amalgamation_file.write(b"</body></html>")
return (num_succeeding_lines, num_failing_files)
def string_to_fmtstr(x):
config = Struct()
loadini(config, default_config_path())
return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))
def format(self, *args, formatter=formatter, **kwargs):
return pygments.format(self.fmttokens(*args, **kwargs), formatter)
def highlight(self, lines):
tokens = CppLexer().get_tokens("\n".join(lines))
source = pygments.format(tokens, TerminalFormatter())
return source.split("\n")
def cache_tokens(self, addr_map):
for filename, line in addr_map.values():
l.warning('caching file: ' + str(filename) + ' at line: ' +
str(line))
if filename != '??':
if filename not in self.file_read_cache.keys():
self.file_cache[filename] = {}
self.file_read_cache[filename] = {}
try:
lexer = pygments.lexers.get_lexer_for_filename(
str(filename))
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs('.highlight')
with open(str(filename)) as f:
content = f.readlines()
if len(content) < 1000:
content = ''.join(content)
tokens = lexer.get_tokens(content)
source = pygments.format(tokens, html_formatter)
self.file_cache[filename][line] = (css, source)
self.file_read_cache[filename] = (lexer, content,
False)
else:
minl = max(0, line - 30)
maxl = min(len(content), line + 30)
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs('.highlight')
source = pygments.format(
lexer.get_tokens(''.join(content[minl:maxl])),
html_formatter)
self.file_cache[filename][line] = (css, source)
self.file_read_cache[filename] = (lexer, content,
True)
except Exception as e:
print(e)
self.file_cache[filename][line] = (None, None)
self.file_read_cache[filename] = (None, None, False)
else:
lexer, content, is_largefile = self.file_read_cache[
filename]
if content:
try:
if not is_largefile:
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs(
'.highlight')
source = pygments.format(
lexer.get_tokens(content), html_formatter)
self.file_cache[filename][line] = (css, source)
else:
minl = max(0, line - 30)
maxl = min(len(content), line + 30)
formatter_opts = dict(linenos="inline",
linespans="line",
hl_lines=[line - minl])
html_formatter = pygments.formatters.get_formatter_by_name(
"html", **formatter_opts)
css = html_formatter.get_style_defs(
'.highlight')
source = pygments.format(
lexer.get_tokens(''.join(
content[minl:maxl])), html_formatter)
self.file_cache[filename][line] = (css, source)
except Exception as e:
print(e)
self.file_cache[filename][line] = (None, None)
else:
self.file_cache[filename][line] = (None, None)
def colorize(self, *tokens: tuple) -> str:
from pygments import format
return format(tokens, self.formatter)
