class PythonTest(unittest.TestCase):
def setUp(self):
self.lexer = PythonLexer()
def test_cls_builtin(self):
"""
Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo
"""
fragment = 'class TestClass():\n @classmethod\n def hello(cls):\n pass\n'
tokens = [
(Token.Keyword, 'class'),
(Token.Text, ' '),
(Token.Name.Class, 'TestClass'),
(Token.Punctuation, '('),
(Token.Punctuation, ')'),
(Token.Punctuation, ':'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Name.Decorator, '@classmethod'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Keyword, 'def'),
(Token.Text, ' '),
(Token.Name.Function, 'hello'),
(Token.Punctuation, '('),
(Token.Name.Builtin.Pseudo, 'cls'),
(Token.Punctuation, ')'),
(Token.Punctuation, ':'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Keyword, 'pass'),
(Token.Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class PygmentsHighlighter(object):
""" highlight python code with a QSyntaxHighlighter,
callable class (e.g. function with a state) to """
def __init__(self):
""" constructor """
self._lexer = PythonLexer()
self._formatter = HtmlFormatter()
self._document = QtGui.QTextDocument()
self._document.setDefaultStyleSheet(self._formatter.get_style_defs())
self._format_cache = dict()
def __call__(self, code):
""" makes this class callable, actually do the highlightning """
index = 0
for token, text in self._lexer.get_tokens(code):
length = len(text)
char_format = self._get_format(token)
pygmentsHighlighter._setFormat(index, length, char_format)
index += length
def _get_format(self, token):
""" get the QTextCharFormat for a token """
if token in self._format_cache:
return self._format_cache[token]
# get format from document
code, html = next(self._formatter._format_lines([(token, u"dummy")]))
self._document.setHtml(html)
char_format = QtGui.QTextCursor(self._document).charFormat()
# cache result
self._format_cache[token] = char_format
return char_format
class PygmentsHighlighter(object):
""" highlight python code with a QSyntaxHighlighter,
callable class (e.g. function with a state) to """
def __init__(self):
""" constructor """
self._lexer = PythonLexer()
self._formatter = HtmlFormatter()
self._document = QtGui.QTextDocument()
self._document.setDefaultStyleSheet(self._formatter.get_style_defs())
self._format_cache = dict()
def __call__(self, code):
""" makes this class callable, actually do the highlightning """
index = 0
for token, text in self._lexer.get_tokens(code):
length = len(text)
char_format = self._get_format(token)
pygmentsHighlighter._setFormat(index, length, char_format)
index += length
def _get_format(self, token):
""" get the QTextCharFormat for a token """
if token in self._format_cache:
return self._format_cache[token]
# get format from document
code, html = self._formatter._format_lines([(token, u'dummy')]).next()
self._document.setHtml(html)
char_format = QtGui.QTextCursor(self._document).charFormat()
# cache result
self._format_cache[token] = char_format
return char_format
def python_line_tokens(code_lines, blank_lines=False):
from pygments.lexers import PythonLexer
lexer = PythonLexer()
code_str = "".join(code_lines)
all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
line_tokens = []
current_line = []
for t in all_tokens:
if t[1] == u"\n":
line_tokens.append(current_line)
current_line = []
else:
current_line.append(t)
rows = []
for i, tokens in enumerate(line_tokens):
# Check for blank line
line_str = code_lines[i].rstrip()
if (not blank_lines) and len(line_str.strip()) == 0:
continue
for t in tokens:
kind, value = str(t[0]), t[1]
yield line_str, i, kind, value, t
class PythonTest(unittest.TestCase):
def setUp(self):
self.lexer = PythonLexer()
def test_cls_builtin(self):
"""
Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo
"""
fragment = 'class TestClass():\n @classmethod\n def hello(cls):\n pass\n'
tokens = [
(Token.Keyword, 'class'),
(Token.Text, ' '),
(Token.Name.Class, 'TestClass'),
(Token.Punctuation, '('),
(Token.Punctuation, ')'),
(Token.Punctuation, ':'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Name.Decorator, '@classmethod'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Keyword, 'def'),
(Token.Text, ' '),
(Token.Name.Function, 'hello'),
(Token.Punctuation, '('),
(Token.Name.Builtin.Pseudo, 'cls'),
(Token.Punctuation, ')'),
(Token.Punctuation, ':'),
(Token.Text, '\n'),
(Token.Text, ' '),
(Token.Keyword, 'pass'),
(Token.Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def python_token_metrics(code_lines, indent_size=4):
from pygments.lexers import PythonLexer
indent_regex = re.compile(r"^\s*")
lexer = PythonLexer()
code_str = "".join(code_lines)
all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
line_tokens = []
current_line = []
for t in all_tokens:
if t[1] == u"\n":
line_tokens.append(current_line)
current_line = []
else:
current_line.append(t)
rows = []
for i, tokens in enumerate(line_tokens):
line_number = i + 1
# Check for blank line
line_str = code_lines[i].rstrip()
if len(line_str.strip()) == 0:
rows.append([line_number, 0, 0, 0, 0, 0, 0])
continue
assert len(tokens) > 0, "No tokens for line"
num_keywords = 0
num_identifiers = 0
num_operators = 0
line_length = len(line_str)
line_indent = len(indent_regex.findall(line_str)[0]) / indent_size
# Indentation is not considered
line_str_noindent = line_str.lstrip()
line_length_noindent = len(line_str_noindent)
whitespace_prop = line_str_noindent.count(" ") / float(line_length_noindent)
for t in tokens:
kind, value = str(t[0]), t[1]
if kind.startswith(u"Token.Keyword"):
num_keywords += 1
elif kind.startswith(u"Token.Name"):
num_identifiers += 1
elif kind.startswith(u"Token.Operator"):
num_operators += 1
rows.append([line_number, line_length_noindent, num_keywords,
num_identifiers, num_operators, whitespace_prop,
line_indent])
columns = ["line", "line_length", "keywords",
"identifiers", "operators", "whitespace_prop",
"line_indent"]
return pandas.DataFrame(rows, columns=columns)
def _analyse_source_code(self, source_code):
lexer = PythonLexer()
token_source = lexer.get_tokens(source_code)
for token_type, value in token_source:
if len(value) > 3 and value.startswith('gl') and ord('A') <= ord(value[2]) <= ord('Z'):
self.gl_functions.add(value)
elif len(value) > 3 and value.startswith('GL_'):
self.gl_constants.add(value)
class CursesParser:
def __init__(self, makecolors = True, style = standardcols):
if makecolors:
self.makecolorpairs()
if style is None:
style = standardcols
self.style = style
self.lexer = PythonLexer()
@classmethod
def makecolorpairs(cls):
"""Initializes curses for colors, makes a color pair of
(color, defaultbg) for every color, and initializes self.colorpairs as
a dictionary with a color -> colorpair mapping"""
if hasattr(cls, 'colorpairs'):
return cls.colorpairs
curses.start_color()
curses.use_default_colors()
maxcolors = curses.COLORS
maxpairs = curses.COLOR_PAIRS
totalmax = min(maxcolors+1, maxpairs)
cls.colorpairs = {}
for colpr in range(1,totalmax):
if colpr >= maxpairs:
break
col = colpr % maxcolors
curses.init_pair(colpr, col, -1)
cls.colorpairs[col] = curses.color_pair(colpr)
return cls.colorpairs
def get_colors(self, raw):
"""Uses pygments to parse the text, and yields (text, color, attr)
tuples"""
for tkn, txt in self.lexer.get_tokens(raw):
notyielded = True
while notyielded:
if tkn is None:
yield (txt, None, None)
notyielded = False
elif tkn in self.style:
col, attr = self.style[tkn]
yield (txt, col, attr)
notyielded = False
else:
tkn = tkn.parent
def parsetoscr(self, scr, raw):
"""Parses text, and uses scr.addstr to print the text directly."""
self.makecolorpairs()
for (txt, col, attr) in self.get_colors(raw):
fullattr = attr
if attr is None:
fullattr = curses.A_NORMAL
if col is not None: # and col in self.colorpairs:
fullattr |= self.colorpairs[col]
scr.addstr(txt, fullattr)
def testWorksAsExpected(self):
code = '''
""" Increment number of decision points in function."""
#if tok and tok.text in McCabeKeywords:
if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
self.metrics['mccabe'] += 1
'''
result = [(Token.Text, u' '), (Token.Literal.String.Doc, u'""" Increment number of decision points in function."""'), (Token.Text, u'\n'), (Token.Text, u' '), (Token.Comment, u'#if tok and tok.text in McCabeKeywords:'), (Token.Text, u'\n'), (Token.Text, u' '), (Token.Keyword, u'if'), (Token.Text, u' '), (Token.Punctuation, u'('), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'=='), (Token.Text, u' '), (Token.Name, u'b'), (Token.Literal.String, u"'"), (Token.Literal.String, u'Keyword'), (Token.Literal.String, u"'"), (Token.Punctuation, u')'), (Token.Text, u' '), (Token.Operator.Word, u'and'), (Token.Text, u' '), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'1'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator.Word, u'in'), (Token.Text, u' '), (Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'), (Token.Text, u'\n'), (Token.Text, u' '), (Token.Name.Builtin.Pseudo, u'self'), (Token.Operator, u'.'), (Token.Name, u'metrics'), (Token.Punctuation, u'['), (Token.Literal.String, u"'"), (Token.Literal.String, u'mccabe'), (Token.Literal.String, u"'"), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'+'), (Token.Operator, u'='), (Token.Text, u' '), (Token.Literal.Number.Integer, u'1'), (Token.Text, u'\n'), (Token.Text, u' '), (Token.Text, u'\n')]
lex = PythonLexer()
tokenList = lex.get_tokens(code)
self.assertEqual(list(tokenList), result)
def testWorksAsExpected(self):
code = '''
""" Increment number of decision points in function."""
#if tok and tok.text in McCabeKeywords:
if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
self.metrics['mccabe'] += 1
'''
result = [(Token.Text, u' '),
(Token.Literal.String.Doc,
u'""" Increment number of decision points in function."""'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Comment, u'#if tok and tok.text in McCabeKeywords:'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Keyword, u'if'), (Token.Text, u' '),
(Token.Punctuation, u'('), (Token.Name, u'tok'),
(Token.Punctuation, u'['),
(Token.Literal.Number.Integer, u'0'),
(Token.Punctuation, u']'), (Token.Punctuation, u'['),
(Token.Literal.Number.Integer, u'0'),
(Token.Punctuation, u']'), (Token.Text, u' '),
(Token.Operator, u'=='), (Token.Text, u' '),
(Token.Name, u'b'), (Token.Literal.String, u"'"),
(Token.Literal.String, u'Keyword'),
(Token.Literal.String, u"'"), (Token.Punctuation, u')'),
(Token.Text, u' '), (Token.Operator.Word, u'and'),
(Token.Text, u' '), (Token.Name, u'tok'),
(Token.Punctuation, u'['),
(Token.Literal.Number.Integer, u'1'),
(Token.Punctuation, u']'), (Token.Text, u' '),
(Token.Operator.Word, u'in'), (Token.Text, u' '),
(Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Name.Builtin.Pseudo, u'self'), (Token.Operator, u'.'),
(Token.Name, u'metrics'), (Token.Punctuation, u'['),
(Token.Literal.String, u"'"),
(Token.Literal.String, u'mccabe'),
(Token.Literal.String, u"'"), (Token.Punctuation, u']'),
(Token.Text, u' '), (Token.Operator, u'+'),
(Token.Operator, u'='), (Token.Text, u' '),
(Token.Literal.Number.Integer, u'1'), (Token.Text, u'\n'),
(Token.Text, u' '), (Token.Text, u'\n')]
lex = PythonLexer()
tokenList = lex.get_tokens(code)
self.assertEqual(list(tokenList), result)
def get_context(string):
""" Assuming the cursor is at the end of the specified string, get the
context (a list of names) for the symbol at cursor position.
"""
lexer = PythonLexer()
context = []
reversed_tokens = list(lexer.get_tokens(string))
reversed_tokens.reverse()
# Pygments often tacks on a newline when none is specified in the input.
# Remove this newline.
if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
not string.endswith('\n'):
reversed_tokens.pop(0)
current_op = ''
for token, text in reversed_tokens:
if is_token_subtype(token, Token.Name):
# Handle a trailing separator, e.g 'foo.bar.'
if current_op == '.':
if not context:
context.insert(0, '')
# Handle non-separator operators and punction.
elif current_op:
break
context.insert(0, text)
current_op = ''
# Pygments doesn't understand that, e.g., '->' is a single operator
# in C++. This is why we have to build up an operator from
# potentially several tokens.
elif token is Token.Operator or token is Token.Punctuation:
current_op = text + current_op
# Break on anything that is not a Operator, Punctuation, or Name.
else:
break
return context
def get_context(string):
""" Assuming the cursor is at the end of the specified string, get the
context (a list of names) for the symbol at cursor position.
"""
lexer = PythonLexer()
context = []
reversed_tokens = list(lexer.get_tokens(string))
reversed_tokens.reverse()
# Pygments often tacks on a newline when none is specified in the input.
# Remove this newline.
if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
not string.endswith('\n'):
reversed_tokens.pop(0)
current_op = ''
for token, text in reversed_tokens:
if is_token_subtype(token, Token.Name):
# Handle a trailing separator, e.g 'foo.bar.'
if current_op == '.':
if not context:
context.insert(0, '')
# Handle non-separator operators and punction.
elif current_op:
break
context.insert(0, text)
current_op = ''
# Pygments doesn't understand that, e.g., '->' is a single operator
# in C++. This is why we have to build up an operator from
# potentially several tokens.
elif token is Token.Operator or token is Token.Punctuation:
current_op = text + current_op
# Break on anything that is not a Operator, Punctuation, or Name.
else:
break
return context
def test_lexer_on_python_global_code():
code = textwrap.dedent('''
""" Increment number of decision points in function."""
#if tok and tok.text in McCabeKeywords:
if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
self.metrics['mccabe'] += 1
''')
result = [
(Token.Literal.String.Doc,
u'""" Increment number of decision points in function."""'),
(Token.Text, u'\n'),
(Token.Comment.Single, u'#if tok and tok.text in McCabeKeywords:'),
(Token.Text, u'\n'), (Token.Keyword, u'if'), (Token.Text, u' '),
(Token.Punctuation, u'('), (Token.Name, u'tok'),
(Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'),
(Token.Punctuation, u']'), (Token.Punctuation, u'['),
(Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'),
(Token.Text, u' '), (Token.Operator, u'=='), (Token.Text, u' '),
(Token.Literal.String.Affix, u'b'),
(Token.Literal.String.Single, u"'"),
(Token.Literal.String.Single, u'Keyword'),
(Token.Literal.String.Single, u"'"), (Token.Punctuation, u')'),
(Token.Text, u' '), (Token.Operator.Word, u'and'), (Token.Text, u' '),
(Token.Name, u'tok'), (Token.Punctuation, u'['),
(Token.Literal.Number.Integer, u'1'), (Token.Punctuation, u']'),
(Token.Text, u' '), (Token.Operator.Word, u'in'), (Token.Text, u' '),
(Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'),
(Token.Text, u'\n'), (Token.Name.Builtin.Pseudo, u'self'),
(Token.Operator, u'.'), (Token.Name, u'metrics'),
(Token.Punctuation, u'['), (Token.Literal.String.Single, u"'"),
(Token.Literal.String.Single, u'mccabe'),
(Token.Literal.String.Single, u"'"), (Token.Punctuation, u']'),
(Token.Text, u' '), (Token.Operator, u'+'), (Token.Operator, u'='),
(Token.Text, u' '), (Token.Literal.Number.Integer, u'1'),
(Token.Text, u'\n')
]
lex = PythonLexer()
tokenList = lex.get_tokens(code)
# print(list(tokenList))
assert list(tokenList) == result
def test_lexer_on_python_function():
code = textwrap.dedent('''
def my_func(p1, p2):
"""Work the incredible magic."""
return p1 + p2
''')
result = [(Token.Keyword, u'def'), (Token.Text, u' '),
(Token.Name.Function, u'my_func'), (Token.Punctuation, u'('),
(Token.Name, u'p1'), (Token.Punctuation, u','),
(Token.Text, u' '), (Token.Name, u'p2'),
(Token.Punctuation, u')'), (Token.Punctuation, u':'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Literal.String.Doc, u'"""Work the incredible magic."""'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Keyword, u'return'), (Token.Text, u' '),
(Token.Name, u'p1'), (Token.Text, u' '), (Token.Operator, u'+'),
(Token.Text, u' '), (Token.Name, u'p2'), (Token.Text, u'\n')]
lex = PythonLexer()
tokenList = lex.get_tokens(code)
# print(list(tokenList))
assert list(tokenList) == result
def test_lexer_on_python_class():
code = textwrap.dedent('''
class MyClass:
"""Definitely first class!"""
variable = "blah"
def my_func(self):
"""Work the incredible magic."""
print("This is a message.")
''')
result = [(Token.Keyword, u'class'), (Token.Text, u' '),
(Token.Name.Class, u'MyClass'), (Token.Punctuation, u':'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Literal.String.Doc, u'"""Definitely first class!"""'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Name, u'variable'), (Token.Text, u' '),
(Token.Operator, u'='), (Token.Text, u' '),
(Token.Literal.String.Double, u'"'),
(Token.Literal.String.Double, u'blah'),
(Token.Literal.String.Double, u'"'), (Token.Text, u'\n'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Keyword, u'def'), (Token.Text, u' '),
(Token.Name.Function, u'my_func'), (Token.Punctuation, u'('),
(Token.Name.Builtin.Pseudo, u'self'), (Token.Punctuation, u')'),
(Token.Punctuation, u':'), (Token.Text, u'\n'),
(Token.Text, u' '),
(Token.Literal.String.Doc, u'"""Work the incredible magic."""'),
(Token.Text, u'\n'), (Token.Text, u' '),
(Token.Keyword, u'print'), (Token.Punctuation, u'('),
(Token.Literal.String.Double, u'"'),
(Token.Literal.String.Double, u'This is a message.'),
(Token.Literal.String.Double, u'"'), (Token.Punctuation, u')'),
(Token.Text, u'\n')]
lex = PythonLexer()
tokenList = lex.get_tokens(code)
# print(list(tokenList))
assert list(tokenList) == result
def __tokenizeFromText(self, text):
lexer = PythonLexer() # Using pygments Python Lexer
tokens = lexer.get_tokens(text)
tokens = list(tokens) # Convert to tokens to list object
result = []
prev_c = '' # Remember previous category
row = 0 # Remember position of row
col = 0 # Remember position of column
# Simplify tokens using categories and add additional coordinates
for token in tokens:
c = get_category(token)
if (c != None):
# Linefeed detected -> do not append to result but change position
if c == 'L':
row = row + 1 #Increment line position
col = 0 # Set back column after linefeed
# New block detected
elif prev_c == 'L' and c != 'I' and result:
self.blocks.append(Block(result))
result = []
if c != 'L':
result.append((c, row, col,
token[1])) # Append result for single token
col += 1 #Increment column position
if col > self._max_col:
self._max_col = col
prev_c = c
self._max_row = row # Set max row for Tokens instance
# Append last block if result not empty
if result:
self.blocks.append(Block(result))
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
""" Syntax highlighter that uses Pygments for parsing. """
style = DefaultStyle # Could be MonokaiStyle, FriendlyStyle, etc.
def __init__(self, parent, lexer=None):
super(PygmentsHighlighter, self).__init__(parent)
try:
self._lexer = get_lexer_by_name(lexer)
except:
self._lexer = PythonLexer()
# Caches for formats and brushes.
self._brushes = {}
self._formats = {}
@handle_exception_in_method
def highlightBlock(self, qstring):
""" Highlight a block of text.
"""
qstring = compat.unicode(qstring)
prev_data = self.previous_block_data()
if prev_data is not None:
self._lexer._epd_state_stack = prev_data.syntax_stack
elif hasattr(self._lexer, '_epd_state_stack'):
del self._lexer._epd_state_stack
index = 0
# Lex the text using Pygments
for token, text in self._lexer.get_tokens(qstring):
l = len(text)
format = self._get_format(token)
if format is not None:
self.setFormat(index, l, format)
index += l
if hasattr(self._lexer, '_epd_state_stack'):
data = BlockUserData(syntax_stack=self._lexer._epd_state_stack)
self.currentBlock().setUserData(data)
# there is a bug in pyside and it will crash unless we
# hold on to the reference a little longer
data = self.currentBlock().userData()
# Clean up for the next go-round.
del self._lexer._epd_state_stack
def previous_block_data(self):
""" Convenience method for returning the previous block's user data.
"""
return self.currentBlock().previous().userData()
def _get_format(self, token):
""" Returns a QTextCharFormat for token or None.
"""
if token in self._formats:
return self._formats[token]
result = None
for key, value in self.style.style_for_token(token) .items():
if value:
if result is None:
result = QtGui.QTextCharFormat()
if key == 'color':
result.setForeground(self._get_brush(value))
elif key == 'bgcolor':
result.setBackground(self._get_brush(value))
elif key == 'bold':
result.setFontWeight(QtGui.QFont.Bold)
elif key == 'italic':
result.setFontItalic(True)
elif key == 'underline':
result.setUnderlineStyle(
QtGui.QTextCharFormat.SingleUnderline)
elif key == 'sans':
result.setFontStyleHint(QtGui.QFont.SansSerif)
elif key == 'roman':
result.setFontStyleHint(QtGui.QFont.Times)
elif key == 'mono':
result.setFontStyleHint(QtGui.QFont.TypeWriter)
elif key == 'border':
# Borders are normally used for errors. We can't do a border
# so instead we do a wavy underline
result.setUnderlineStyle(
QtGui.QTextCharFormat.WaveUnderline)
result.setUnderlineColor(self._get_color(value))
self._formats[token] = result
return result
def _get_brush(self, color):
""" Returns a brush for the color.
"""
result = self._brushes.get(color)
if result is None:
qcolor = self._get_color(color)
result = QtGui.QBrush(qcolor)
self._brushes[color] = result
return result
def _get_color(self, color):
qcolor = QtGui.QColor()
qcolor.setRgb(int(color[:2], base=16),
int(color[2:4], base=16),
int(color[4:6], base=16))
return qcolor
def _lex_python_result(tb):
" Return token list for Python string. "
lexer = PythonLexer()
return lexer.get_tokens(tb)
def make_code_aois(code_file,
font_size=(14, 25),
line_offset=5,
syntax_categories=SYNTAX_CATEGORIES):
aoi_df = pandas.DataFrame(columns=("aoi_kind", "name", "x", "y", "width",
"height", "note"))
# Needed for syntax-based AOIs
from pygments.lexers import PythonLexer
lexer = PythonLexer()
# Parse each file and generate AOIs
code_lines = code_file.readlines()
code_str = "".join(code_lines)
# Add extra newline token to trigger last AOI block
code_lines += [""]
tokens = list(lexer.get_tokens(code_str,
unfiltered=True)) + [("Token.Text", u"\n")]
col = 0 # Current column
line = 0 # Current line number
block_start = 0 # Current whitespace separated block
last_blank = False # Was last line blank?
block_lines = [] # Lines in current block
for t in tokens:
kind = str(t[0])
val = t[1]
# Check if end of line
if val == u"\n":
line_str = code_lines[line].rstrip()
if len(line_str.strip()) > 0:
# Non-blank line: add AOI for whole line
aoi_df = aoi_df.append(
{
"aoi_kind":
"line",
"name":
"line {0}".format(line + 1),
"x":
0,
"y": (line * font_size[1]) + (line * line_offset) -
(line_offset / 2),
"width":
len(line_str) * font_size[0],
"height":
font_size[1] + line_offset - 1,
"note":
line_str
},
ignore_index=True)
# Add to current block
last_blank = False
block_lines.append(line_str)
else:
# Blank line
if not last_blank:
# Add AOI for whitespace separated block of lines
aoi_df = aoi_df.append(
{
"aoi_kind":
"block",
"name":
"lines {0}-{1}".format(block_start + 1, line + 1),
"x":
0,
"y": (block_start * font_size[1]) +
(block_start * line_offset) - (line_offset / 2),
"width":
max([len(l) for l in block_lines]) * font_size[0],
"height":
len(block_lines) * (font_size[1] + line_offset),
"note":
"\n".join(block_lines)
},
ignore_index=True)
# Reset block variables
last_blank = True
block_lines = []
block_start = line + 1
# Next line
col = 0
line += 1
continue
# Add AOI for syntax token
if (kind == "Token.Text") and (col == 0):
kind += ".Indentation"
aoi_df = aoi_df.append(
{
"aoi_kind":
"syntax",
"name":
syntax_categories[kind],
"x":
col * font_size[0],
"y": (line * font_size[1]) + (line * line_offset) -
(line_offset / 2),
"width":
len(val) * font_size[0],
"height":
font_size[1] + line_offset - 1,
"note":
val
},
ignore_index=True)
col += len(val)
return aoi_df
def python_line_categories(code_lines):
from pygments.lexers import PythonLexer
lexer = PythonLexer()
code_str = "".join(code_lines)
all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
line_tokens = []
current_line = []
for t in all_tokens:
if t[1] == u"\n":
line_tokens.append(current_line)
current_line = []
else:
current_line.append(t)
line_categories = []
for i, tokens in enumerate(line_tokens):
# Check for blank line
line_str = code_lines[i].rstrip()
if len(line_str.strip()) == 0:
line_categories.append(["blank line"])
continue
assert len(tokens) > 0, "No tokens for line"
categories = []
last_kind, last_value = None, None
for t in tokens:
kind, value = str(t[0]), t[1]
if kind == u"Token.Keyword" and value == u"def":
categories.append("function definition")
elif kind == u"Token.Keyword" and value == u"if":
categories.append("if statement")
elif kind == u"Token.Keyword" and value == u"for":
categories.append("for loop")
elif kind == u"Token.Keyword" and value == u"return":
categories.append("return statement")
elif kind == u"Token.Keyword" and value == u"print":
categories.append("print statement")
elif kind == u"Token.Keyword" and value == u"class":
categories.append("class definition")
elif kind == u"Token.Operator" and value == u"=":
categories.append("assignment")
elif kind == u"Token.Operator" and value == u".":
categories.append("object access")
elif kind == u"Token.Operator" and value in [u"+", u"*"]:
categories.append("mathematical operation")
elif last_kind == u"Token.Operator" and last_value == u"-" and kind == "Token.Whitespace":
categories.append("mathematical operation")
elif kind == u"Token.Operator" and value in [u"<", u">"]:
categories.append("comparison")
elif last_kind == u"Token.Name" and kind == "Token.Punctuation" and value == u"(":
categories.append("function call")
elif kind == "Token.Punctuation" and value == u"[":
categories.append("list creation")
last_kind, last_value = kind, value
if len(categories) == 0:
categories.append("unknown")
line_categories.append(set(categories))
return line_categories
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
""" Syntax highlighter that uses Pygments for parsing. """
def __init__(self, parent, lexer=None):
super(PygmentsHighlighter, self).__init__(parent)
try:
self._lexer = get_lexer_by_name(lexer)
except:
self._lexer = PythonLexer()
self._style = DefaultStyle
# Caches for formats and brushes.
self._brushes = {}
self._formats = {}
def highlightBlock(self, qstring):
""" Highlight a block of text.
"""
qstring = str(qstring)
prev_data = self.previous_block_data()
if prev_data is not None:
self._lexer._saved_state_stack = prev_data.syntax_stack
elif hasattr(self._lexer, "_saved_state_stack"):
del self._lexer._saved_state_stack
index = 0
# Lex the text using Pygments
for token, text in self._lexer.get_tokens(qstring):
l = len(text)
format = self._get_format(token)
if format is not None:
self.setFormat(index, l, format)
index += l
if hasattr(self._lexer, "_saved_state_stack"):
data = BlockUserData(syntax_stack=self._lexer._saved_state_stack)
self.currentBlock().setUserData(data)
# there is a bug in pyside and it will crash unless we
# hold on to the reference a little longer
data = self.currentBlock().userData()
# Clean up for the next go-round.
del self._lexer._saved_state_stack
def previous_block_data(self):
""" Convenience method for returning the previous block's user data.
"""
return self.currentBlock().previous().userData()
def _get_format(self, token):
""" Returns a QTextCharFormat for token or None.
"""
if token in self._formats:
return self._formats[token]
result = None
while not self._style.styles_token(token):
token = token.parent
for key, value in self._style.style_for_token(token).items():
if value:
if result is None:
result = QtGui.QTextCharFormat()
if key == "color":
result.setForeground(self._get_brush(value))
elif key == "bgcolor":
result.setBackground(self._get_brush(value))
elif key == "bold":
result.setFontWeight(QtGui.QFont.Bold)
elif key == "italic":
result.setFontItalic(True)
elif key == "underline":
result.setUnderlineStyle(
QtGui.QTextCharFormat.SingleUnderline)
elif key == "sans":
result.setFontStyleHint(QtGui.QFont.SansSerif)
elif key == "roman":
result.setFontStyleHint(QtGui.QFont.Times)
elif key == "mono":
result.setFontStyleHint(QtGui.QFont.TypeWriter)
elif key == "border":
# Borders are normally used for errors. We can't do a border
# so instead we do a wavy underline
result.setUnderlineStyle(
QtGui.QTextCharFormat.WaveUnderline)
result.setUnderlineColor(self._get_color(value))
self._formats[token] = result
return result
def _get_brush(self, color):
""" Returns a brush for the color.
"""
result = self._brushes.get(color)
if result is None:
qcolor = self._get_color(color)
result = QtGui.QBrush(qcolor)
self._brushes[color] = result
return result
def _get_color(self, color):
qcolor = QtGui.QColor()
qcolor.setRgb(
int(color[:2], base=16),
int(color[2:4], base=16),
int(color[4:6], base=16),
)
return qcolor
def make_code_aois(code_file, font_size=(14, 25), line_offset=5,
syntax_categories=SYNTAX_CATEGORIES):
aoi_df = pandas.DataFrame(columns=("aoi_kind", "name",
"x", "y", "width", "height", "note"))
# Needed for syntax-based AOIs
from pygments.lexers import PythonLexer
lexer = PythonLexer()
# Parse each file and generate AOIs
code_lines = code_file.readlines()
code_str = "".join(code_lines)
# Add extra newline token to trigger last AOI block
code_lines += [""]
tokens = list(lexer.get_tokens(code_str, unfiltered=True)) + [("Token.Text", u"\n")]
col = 0 # Current column
line = 0 # Current line number
block_start = 0 # Current whitespace separated block
last_blank = False # Was last line blank?
block_lines = [] # Lines in current block
for t in tokens:
kind = str(t[0])
val = t[1]
# Check if end of line
if val == u"\n":
line_str = code_lines[line].rstrip()
if len(line_str.strip()) > 0:
# Non-blank line: add AOI for whole line
aoi_df = aoi_df.append({
"aoi_kind" : "line",
"name" : "line {0}".format(line + 1),
"x" : 0,
"y" : (line * font_size[1]) + (line * line_offset) - (line_offset / 2),
"width" : len(line_str) * font_size[0],
"height" : font_size[1] + line_offset - 1,
"note" : line_str
}, ignore_index=True)
# Add to current block
last_blank = False
block_lines.append(line_str)
else:
# Blank line
if not last_blank:
# Add AOI for whitespace separated block of lines
aoi_df = aoi_df.append({
"aoi_kind" : "block",
"name" : "lines {0}-{1}".format(block_start + 1, line + 1),
"x" : 0,
"y" : (block_start * font_size[1]) + (block_start * line_offset) - (line_offset / 2),
"width" : max([len(l) for l in block_lines]) * font_size[0],
"height" : len(block_lines) * (font_size[1] + line_offset),
"note" : "\n".join(block_lines)
}, ignore_index=True)
# Reset block variables
last_blank = True
block_lines = []
block_start = line + 1
# Next line
col = 0
line += 1
continue
# Add AOI for syntax token
if (kind == "Token.Text") and (col == 0):
kind += ".Indentation"
aoi_df = aoi_df.append({
"aoi_kind" : "syntax",
"name" : syntax_categories[kind],
"x" : col * font_size[0],
"y" : (line * font_size[1]) + (line * line_offset) - (line_offset / 2),
"width" : len(val) * font_size[0],
"height" : font_size[1] + line_offset - 1,
"note" : val
}, ignore_index=True)
col += len(val)
return aoi_df
def _lex_python_result(tb):
" Return token list for Python string. "
lexer = PythonLexer()
return lexer.get_tokens(tb)
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
""" Syntax highlighter that uses Pygments for parsing. """
#---------------------------------------------------------------------------
# 'QSyntaxHighlighter' interface
#---------------------------------------------------------------------------
def __init__(self, parent, lexer=None):
super(PygmentsHighlighter, self).__init__(parent)
self._document = self.document()
self._formatter = HtmlFormatter(nowrap=True)
self.set_style('default')
if lexer is not None:
self._lexer = lexer
else:
if PY3:
self._lexer = Python3Lexer()
else:
self._lexer = PythonLexer()
def highlightBlock(self, string):
""" Highlight a block of text.
"""
prev_data = self.currentBlock().previous().userData()
if prev_data is not None:
self._lexer._saved_state_stack = prev_data.syntax_stack
elif hasattr(self._lexer, '_saved_state_stack'):
del self._lexer._saved_state_stack
# Lex the text using Pygments
index = 0
for token, text in self._lexer.get_tokens(string):
length = qstring_length(text)
self.setFormat(index, length, self._get_format(token))
index += length
if hasattr(self._lexer, '_saved_state_stack'):
data = PygmentsBlockUserData(
syntax_stack=self._lexer._saved_state_stack)
self.currentBlock().setUserData(data)
# Clean up for the next go-round.
del self._lexer._saved_state_stack
#---------------------------------------------------------------------------
# 'PygmentsHighlighter' interface
#---------------------------------------------------------------------------
def set_style(self, style):
""" Sets the style to the specified Pygments style.
"""
if isinstance(style, string_types):
style = get_style_by_name(style)
self._style = style
self._clear_caches()
def set_style_sheet(self, stylesheet):
""" Sets a CSS stylesheet. The classes in the stylesheet should
correspond to those generated by:
pygmentize -S <style> -f html
Note that 'set_style' and 'set_style_sheet' completely override each
other, i.e. they cannot be used in conjunction.
"""
self._document.setDefaultStyleSheet(stylesheet)
self._style = None
self._clear_caches()
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _clear_caches(self):
""" Clear caches for brushes and formats.
"""
self._brushes = {}
self._formats = {}
def _get_format(self, token):
""" Returns a QTextCharFormat for token or None.
"""
if token in self._formats:
return self._formats[token]
if self._style is None:
result = self._get_format_from_document(token, self._document)
else:
result = self._get_format_from_style(token, self._style)
self._formats[token] = result
return result
def _get_format_from_document(self, token, document):
""" Returns a QTextCharFormat for token by
"""
code, html = next(self._formatter._format_lines([(token, u'dummy')]))
self._document.setHtml(html)
return QtGui.QTextCursor(self._document).charFormat()
def _get_format_from_style(self, token, style):
""" Returns a QTextCharFormat for token by reading a Pygments style.
"""
result = QtGui.QTextCharFormat()
for key, value in style.style_for_token(token).items():
if value:
if key == 'color':
result.setForeground(self._get_brush(value))
elif key == 'bgcolor':
result.setBackground(self._get_brush(value))
elif key == 'bold':
result.setFontWeight(QtGui.QFont.Bold)
elif key == 'italic':
result.setFontItalic(True)
elif key == 'underline':
result.setUnderlineStyle(
QtGui.QTextCharFormat.SingleUnderline)
elif key == 'sans':
result.setFontStyleHint(QtGui.QFont.SansSerif)
elif key == 'roman':
result.setFontStyleHint(QtGui.QFont.Times)
elif key == 'mono':
result.setFontStyleHint(QtGui.QFont.TypeWriter)
return result
def _get_brush(self, color):
""" Returns a brush for the color.
"""
result = self._brushes.get(color)
if result is None:
qcolor = self._get_color(color)
result = QtGui.QBrush(qcolor)
self._brushes[color] = result
return result
def _get_color(self, color):
""" Returns a QColor built from a Pygments color string.
"""
qcolor = QtGui.QColor()
qcolor.setRgb(int(color[:2], base=16),
int(color[2:4], base=16),
int(color[4:6], base=16))
return qcolor
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
""" Syntax highlighter that uses Pygments for parsing. """
#---------------------------------------------------------------------------
# 'QSyntaxHighlighter' interface
#---------------------------------------------------------------------------
def __init__(self, parent, lexer=None):
super(PygmentsHighlighter, self).__init__(parent)
self._document = self.document()
self._formatter = HtmlFormatter(nowrap=True)
self.set_style('default')
if lexer is not None:
self._lexer = lexer
else:
if PY3:
self._lexer = Python3Lexer()
else:
self._lexer = PythonLexer()
def highlightBlock(self, string):
""" Highlight a block of text.
"""
prev_data = self.currentBlock().previous().userData()
if prev_data is not None:
self._lexer._saved_state_stack = prev_data.syntax_stack
elif hasattr(self._lexer, '_saved_state_stack'):
del self._lexer._saved_state_stack
# Lex the text using Pygments
index = 0
for token, text in self._lexer.get_tokens(string):
length = len(text)
self.setFormat(index, length, self._get_format(token))
index += length
if hasattr(self._lexer, '_saved_state_stack'):
data = PygmentsBlockUserData(
syntax_stack=self._lexer._saved_state_stack)
self.currentBlock().setUserData(data)
# Clean up for the next go-round.
del self._lexer._saved_state_stack
#---------------------------------------------------------------------------
# 'PygmentsHighlighter' interface
#---------------------------------------------------------------------------
def set_style(self, style):
""" Sets the style to the specified Pygments style.
"""
if isinstance(style, string_types):
style = get_style_by_name(style)
self._style = style
self._clear_caches()
def set_style_sheet(self, stylesheet):
""" Sets a CSS stylesheet. The classes in the stylesheet should
correspond to those generated by:
pygmentize -S <style> -f html
Note that 'set_style' and 'set_style_sheet' completely override each
other, i.e. they cannot be used in conjunction.
"""
self._document.setDefaultStyleSheet(stylesheet)
self._style = None
self._clear_caches()
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _clear_caches(self):
""" Clear caches for brushes and formats.
"""
self._brushes = {}
self._formats = {}
def _get_format(self, token):
""" Returns a QTextCharFormat for token or None.
"""
if token in self._formats:
return self._formats[token]
if self._style is None:
result = self._get_format_from_document(token, self._document)
else:
result = self._get_format_from_style(token, self._style)
self._formats[token] = result
return result
def _get_format_from_document(self, token, document):
""" Returns a QTextCharFormat for token by
"""
code, html = next(self._formatter._format_lines([(token, u'dummy')]))
self._document.setHtml(html)
return QtGui.QTextCursor(self._document).charFormat()
def _get_format_from_style(self, token, style):
""" Returns a QTextCharFormat for token by reading a Pygments style.
"""
result = QtGui.QTextCharFormat()
for key, value in style.style_for_token(token).items():
if value:
if key == 'color':
result.setForeground(self._get_brush(value))
elif key == 'bgcolor':
result.setBackground(self._get_brush(value))
elif key == 'bold':
result.setFontWeight(QtGui.QFont.Bold)
elif key == 'italic':
result.setFontItalic(True)
elif key == 'underline':
result.setUnderlineStyle(
QtGui.QTextCharFormat.SingleUnderline)
elif key == 'sans':
result.setFontStyleHint(QtGui.QFont.SansSerif)
elif key == 'roman':
result.setFontStyleHint(QtGui.QFont.Times)
elif key == 'mono':
result.setFontStyleHint(QtGui.QFont.TypeWriter)
return result
def _get_brush(self, color):
""" Returns a brush for the color.
"""
result = self._brushes.get(color)
if result is None:
qcolor = self._get_color(color)
result = QtGui.QBrush(qcolor)
self._brushes[color] = result
return result
def _get_color(self, color):
""" Returns a QColor built from a Pygments color string.
"""
qcolor = QtGui.QColor()
qcolor.setRgb(int(color[:2], base=16),
int(color[2:4], base=16),
int(color[4:6], base=16))
return qcolor
"""
import re
import sys
from pygments.lexers import PythonLexer
from pygments.token import Punctuation, Comment
if __name__ == "__main__":
lexer = PythonLexer()
pattern = re.compile(u".*(given|and_|when|then)\((.+)")
steps = []
print "# -*- coding: UTF-8 -*-"
print "from pyfeature import step"
for l in sys.stdin.xreadlines():
m = pattern.match(l)
if m:
s = lexer.get_tokens(m.group(2))
desc = ""
for t in s:
if t[0] != Punctuation and t[1] != u",":
desc += "".join(chr(c) for c in [ord(b) for b in t[1]])
else:
break
steps.append([desc.strip()])
omit_next_token = False
for t in s:
if t[1] == "=":
omit_next_token = True
elif not omit_next_token and t[1] != "," and t[1] != ")" and t[0] != Comment:
steps[-1].append(t[1])
else:
omit_next_token = False
class Console(QTextEdit):
running = pyqtSignal()
running_done = pyqtSignal()
@property
def doc(self):
return self.document()
@property
def cursor(self):
return self.textCursor()
@property
def root_frame(self):
return self.doc.rootFrame()
def unhandled_exception(self, type, value, tb):
if type == KeyboardInterrupt:
return
try:
sio = StringIO.StringIO()
traceback.print_exception(type, value, tb, file=sio)
fe = sio.getvalue()
prints(fe)
try:
val = unicode(value)
except:
val = repr(value)
msg = '<b>%s</b>:'%type.__name__ + val
error_dialog(self, _('ERROR: Unhandled exception'), msg,
det_msg=fe, show=True)
except BaseException:
pass
def __init__(self,
prompt='>>> ',
continuation='... ',
parent=None):
QTextEdit.__init__(self, parent)
self.shutting_down = False
self.compiler = CommandCompiler()
self.buf = self.old_buf = []
self.history = History([''], dynamic.get('console_history', []))
self.prompt_frame = None
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
self.prompt_frame_format.setBorder(1)
self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
self.prompt_len = len(prompt)
self.doc.setMaximumBlockCount(int(prefs['scrollback']))
self.lexer = PythonLexer(ensurenl=False)
self.tb_lexer = PythonTracebackLexer()
self.context_menu = cm = QMenu(self) # {{{
cm.theme = ThemeMenu(cm)
# }}}
self.formatter = Formatter(prompt, continuation, style=prefs['theme'])
p = QPalette()
p.setColor(p.Base, QColor(self.formatter.background_color))
p.setColor(p.Text, QColor(self.formatter.color))
self.setPalette(p)
self.key_dispatcher = { # {{{
Qt.Key_Enter : self.enter_pressed,
Qt.Key_Return : self.enter_pressed,
Qt.Key_Up : self.up_pressed,
Qt.Key_Down : self.down_pressed,
Qt.Key_Home : self.home_pressed,
Qt.Key_End : self.end_pressed,
Qt.Key_Left : self.left_pressed,
Qt.Key_Right : self.right_pressed,
Qt.Key_Backspace : self.backspace_pressed,
Qt.Key_Delete : self.delete_pressed,
} # }}}
motd = textwrap.dedent('''\
# Python {0}
# {1} {2}
'''.format(sys.version.splitlines()[0], __appname__,
__version__))
sys.excepthook = self.unhandled_exception
self.controllers = []
QTimer.singleShot(0, self.launch_controller)
with EditBlock(self.cursor):
self.render_block(motd)
def shutdown(self):
dynamic.set('console_history', self.history.serialize())
self.shutting_down = True
for c in self.controllers:
c.kill()
def contextMenuEvent(self, event):
self.context_menu.popup(event.globalPos())
event.accept()
# Controller management {{{
@property
def controller(self):
return self.controllers[-1]
def no_controller_error(self):
error_dialog(self, _('No interpreter'),
_('No active interpreter found. Try restarting the'
' console'), show=True)
def launch_controller(self, *args):
c = Controller(self)
c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
c.interpreter_died.connect(self.interpreter_died,
type=Qt.QueuedConnection)
c.interpreter_done.connect(self.execution_done)
self.controllers.append(c)
def interpreter_died(self, controller, returncode):
if not self.shutting_down and controller.current_command is not None:
error_dialog(self, _('Interpreter died'),
_('Interpreter dies while executing a command. To see '
'the command, click Show details'),
det_msg=controller.current_command, show=True)
def execute(self, prompt_lines):
c = self.root_frame.lastCursorPosition()
self.setTextCursor(c)
self.old_prompt_frame = self.prompt_frame
self.prompt_frame = None
self.old_buf = self.buf
self.buf = []
self.running.emit()
self.controller.runsource('\n'.join(prompt_lines))
def execution_done(self, controller, ret):
if controller is self.controller:
self.running_done.emit()
if ret: # Incomplete command
self.buf = self.old_buf
self.prompt_frame = self.old_prompt_frame
c = self.prompt_frame.lastCursorPosition()
c.insertBlock()
self.setTextCursor(c)
else: # Command completed
try:
self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
except RuntimeError:
# Happens if enough lines of output that the old
# frame was deleted
pass
self.render_current_prompt()
# }}}
# Prompt management {{{
@dynamic_property
def cursor_pos(self):
doc = '''
The cursor position in the prompt has the form (row, col).
row starts at 0 for the first line
col is 0 if the cursor is at the start of the line, 1 if it is after
the first character, n if it is after the nth char.
'''
def fget(self):
if self.prompt_frame is not None:
pos = self.cursor.position()
it = self.prompt_frame.begin()
lineno = 0
while not it.atEnd():
bl = it.currentBlock()
if bl.contains(pos):
return (lineno, pos - bl.position())
it += 1
lineno += 1
return (-1, -1)
def fset(self, val):
row, col = val
if self.prompt_frame is not None:
it = self.prompt_frame.begin()
lineno = 0
while not it.atEnd():
if lineno == row:
c = self.cursor
c.setPosition(it.currentBlock().position())
c.movePosition(c.NextCharacter, n=col)
self.setTextCursor(c)
break
it += 1
lineno += 1
return property(fget=fget, fset=fset, doc=doc)
def move_cursor_to_prompt(self):
if self.prompt_frame is not None and self.cursor_pos[0] < 0:
c = self.prompt_frame.lastCursorPosition()
self.setTextCursor(c)
def prompt(self, strip_prompt_strings=True):
if not self.prompt_frame:
yield u'' if strip_prompt_strings else self.formatter.prompt
else:
it = self.prompt_frame.begin()
while not it.atEnd():
bl = it.currentBlock()
t = unicode(bl.text())
if strip_prompt_strings:
t = t[self.prompt_len:]
yield t
it += 1
def set_prompt(self, lines):
self.render_current_prompt(lines)
def clear_current_prompt(self):
if self.prompt_frame is None:
c = self.root_frame.lastCursorPosition()
self.prompt_frame = c.insertFrame(self.prompt_frame_format)
self.setTextCursor(c)
else:
c = self.prompt_frame.firstCursorPosition()
self.setTextCursor(c)
c.setPosition(self.prompt_frame.lastPosition(), c.KeepAnchor)
c.removeSelectedText()
c.setPosition(self.prompt_frame.firstPosition())
def render_current_prompt(self, lines=None, restore_cursor=False):
row, col = self.cursor_pos
cp = list(self.prompt()) if lines is None else lines
self.clear_current_prompt()
for i, line in enumerate(cp):
start = i == 0
end = i == len(cp) - 1
self.formatter.render_prompt(not start, self.cursor)
self.formatter.render(self.lexer.get_tokens(line), self.cursor)
if not end:
self.cursor.insertBlock()
if row > -1 and restore_cursor:
self.cursor_pos = (row, col)
self.ensureCursorVisible()
# }}}
# Non-prompt Rendering {{{
def render_block(self, text, restore_prompt=True):
self.formatter.render(self.lexer.get_tokens(text), self.cursor)
self.cursor.insertBlock()
self.cursor.movePosition(self.cursor.End)
if restore_prompt:
self.render_current_prompt()
def show_error(self, is_syntax_err, tb, controller=None):
if self.prompt_frame is not None:
# At a prompt, so redirect output
return prints(tb, end='')
try:
self.buf.append(tb)
if is_syntax_err:
self.formatter.render_syntax_error(tb, self.cursor)
else:
self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
except:
prints(tb, end='')
self.ensureCursorVisible()
QApplication.processEvents()
def show_output(self, raw, which='stdout', controller=None):
def do_show():
try:
self.buf.append(raw)
self.formatter.render_raw(raw, self.cursor)
except:
import traceback
prints(traceback.format_exc())
prints(raw, end='')
if self.prompt_frame is not None:
with Prepender(self):
do_show()
else:
do_show()
self.ensureCursorVisible()
QApplication.processEvents()
# }}}
# Keyboard management {{{
def keyPressEvent(self, ev):
text = unicode(ev.text())
key = ev.key()
action = self.key_dispatcher.get(key, None)
if callable(action):
action()
elif key in (Qt.Key_Escape,):
QTextEdit.keyPressEvent(self, ev)
elif text:
self.text_typed(text)
else:
QTextEdit.keyPressEvent(self, ev)
def left_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if pos > self.prompt_len:
c = self.cursor
c.movePosition(c.PreviousCharacter)
self.setTextCursor(c)
elif lineno > 0:
c = self.cursor
c.movePosition(c.Up)
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def up_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if lineno == 0:
b = self.history.back()
if b is not None:
self.set_prompt(b)
else:
c = self.cursor
c.movePosition(c.Up)
self.setTextCursor(c)
self.ensureCursorVisible()
def backspace_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if pos > self.prompt_len:
self.cursor.deletePreviousChar()
elif lineno > 0:
c = self.cursor
c.movePosition(c.Up)
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def delete_pressed(self):
self.cursor.deleteChar()
self.ensureCursorVisible()
def right_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
c = self.cursor
cp = list(self.prompt(False))
if pos < len(cp[lineno]):
c.movePosition(c.NextCharacter)
elif lineno < len(cp)-1:
c.movePosition(c.NextCharacter, n=1+self.prompt_len)
self.setTextCursor(c)
self.ensureCursorVisible()
def down_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
c = self.cursor
cp = list(self.prompt(False))
if lineno >= len(cp) - 1:
b = self.history.forward()
if b is not None:
self.set_prompt(b)
else:
c = self.cursor
c.movePosition(c.Down)
self.setTextCursor(c)
self.ensureCursorVisible()
def home_pressed(self):
if self.prompt_frame is not None:
mods = QApplication.keyboardModifiers()
ctrl = bool(int(mods & Qt.CTRL))
if ctrl:
self.cursor_pos = (0, self.prompt_len)
else:
c = self.cursor
c.movePosition(c.StartOfLine)
c.movePosition(c.NextCharacter, n=self.prompt_len)
self.setTextCursor(c)
self.ensureCursorVisible()
def end_pressed(self):
if self.prompt_frame is not None:
mods = QApplication.keyboardModifiers()
ctrl = bool(int(mods & Qt.CTRL))
if ctrl:
self.cursor_pos = (len(list(self.prompt()))-1, self.prompt_len)
c = self.cursor
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def enter_pressed(self):
if self.prompt_frame is None:
return
if not self.controller.is_alive:
return self.no_controller_error()
cp = list(self.prompt())
if cp[0]:
try:
ret = self.compiler('\n'.join(cp))
except:
pass
else:
if ret is None:
c = self.prompt_frame.lastCursorPosition()
c.insertBlock()
self.setTextCursor(c)
self.render_current_prompt()
return
else:
self.history.enter(cp)
self.execute(cp)
def text_typed(self, text):
if self.prompt_frame is not None:
self.move_cursor_to_prompt()
self.cursor.insertText(text)
self.render_current_prompt(restore_cursor=True)
self.history.current = list(self.prompt())
class Console(QTextEdit):
running = pyqtSignal()
running_done = pyqtSignal()
@property
def doc(self):
return self.document()
@property
def cursor(self):
return self.textCursor()
@property
def root_frame(self):
return self.doc.rootFrame()
def unhandled_exception(self, type, value, tb):
if type == KeyboardInterrupt:
return
try:
sio = StringIO.StringIO()
traceback.print_exception(type, value, tb, file=sio)
fe = sio.getvalue()
prints(fe)
try:
val = unicode(value)
except:
val = repr(value)
msg = '<b>%s</b>:'%type.__name__ + val
error_dialog(self, _('ERROR: Unhandled exception'), msg,
det_msg=fe, show=True)
except BaseException:
pass
def __init__(self,
prompt='>>> ',
continuation='... ',
parent=None):
QTextEdit.__init__(self, parent)
self.shutting_down = False
self.compiler = CommandCompiler()
self.buf = self.old_buf = []
self.history = History([''], dynamic.get('console_history', []))
self.prompt_frame = None
self.allow_output = False
self.prompt_frame_format = QTextFrameFormat()
self.prompt_frame_format.setBorder(1)
self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
self.prompt_len = len(prompt)
self.doc.setMaximumBlockCount(int(prefs['scrollback']))
self.lexer = PythonLexer(ensurenl=False)
self.tb_lexer = PythonTracebackLexer()
self.context_menu = cm = QMenu(self) # {{{
cm.theme = ThemeMenu(cm)
# }}}
self.formatter = Formatter(prompt, continuation, style=prefs['theme'])
p = QPalette()
p.setColor(p.Base, QColor(self.formatter.background_color))
p.setColor(p.Text, QColor(self.formatter.color))
self.setPalette(p)
self.key_dispatcher = { # {{{
Qt.Key_Enter : self.enter_pressed,
Qt.Key_Return : self.enter_pressed,
Qt.Key_Up : self.up_pressed,
Qt.Key_Down : self.down_pressed,
Qt.Key_Home : self.home_pressed,
Qt.Key_End : self.end_pressed,
Qt.Key_Left : self.left_pressed,
Qt.Key_Right : self.right_pressed,
Qt.Key_Backspace : self.backspace_pressed,
Qt.Key_Delete : self.delete_pressed,
} # }}}
motd = textwrap.dedent('''\
# Python {0}
# {1} {2}
'''.format(sys.version.splitlines()[0], __appname__,
__version__))
sys.excepthook = self.unhandled_exception
self.controllers = []
QTimer.singleShot(0, self.launch_controller)
with EditBlock(self.cursor):
self.render_block(motd)
def shutdown(self):
dynamic.set('console_history', self.history.serialize())
self.shutting_down = True
for c in self.controllers:
c.kill()
def contextMenuEvent(self, event):
self.context_menu.popup(event.globalPos())
event.accept()
# Controller management {{{
@property
def controller(self):
return self.controllers[-1]
def no_controller_error(self):
error_dialog(self, _('No interpreter'),
_('No active interpreter found. Try restarting the'
' console'), show=True)
def launch_controller(self, *args):
c = Controller(self)
c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
c.interpreter_died.connect(self.interpreter_died,
type=Qt.QueuedConnection)
c.interpreter_done.connect(self.execution_done)
self.controllers.append(c)
def interpreter_died(self, controller, returncode):
if not self.shutting_down and controller.current_command is not None:
error_dialog(self, _('Interpreter died'),
_('Interpreter dies while executing a command. To see '
'the command, click Show details'),
det_msg=controller.current_command, show=True)
def execute(self, prompt_lines):
c = self.root_frame.lastCursorPosition()
self.setTextCursor(c)
self.old_prompt_frame = self.prompt_frame
self.prompt_frame = None
self.old_buf = self.buf
self.buf = []
self.running.emit()
self.controller.runsource('\n'.join(prompt_lines))
def execution_done(self, controller, ret):
if controller is self.controller:
self.running_done.emit()
if ret: # Incomplete command
self.buf = self.old_buf
self.prompt_frame = self.old_prompt_frame
c = self.prompt_frame.lastCursorPosition()
c.insertBlock()
self.setTextCursor(c)
else: # Command completed
try:
self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
except RuntimeError:
# Happens if enough lines of output that the old
# frame was deleted
pass
self.render_current_prompt()
# }}}
# Prompt management {{{
@dynamic_property
def cursor_pos(self):
doc = '''
The cursor position in the prompt has the form (row, col).
row starts at 0 for the first line
col is 0 if the cursor is at the start of the line, 1 if it is after
the first character, n if it is after the nth char.
'''
def fget(self):
if self.prompt_frame is not None:
pos = self.cursor.position()
it = self.prompt_frame.begin()
lineno = 0
while not it.atEnd():
bl = it.currentBlock()
if bl.contains(pos):
return (lineno, pos - bl.position())
it += 1
lineno += 1
return (-1, -1)
def fset(self, val):
row, col = val
if self.prompt_frame is not None:
it = self.prompt_frame.begin()
lineno = 0
while not it.atEnd():
if lineno == row:
c = self.cursor
c.setPosition(it.currentBlock().position())
c.movePosition(c.NextCharacter, n=col)
self.setTextCursor(c)
break
it += 1
lineno += 1
return property(fget=fget, fset=fset, doc=doc)
def move_cursor_to_prompt(self):
if self.prompt_frame is not None and self.cursor_pos[0] < 0:
c = self.prompt_frame.lastCursorPosition()
self.setTextCursor(c)
def prompt(self, strip_prompt_strings=True):
if not self.prompt_frame:
yield u'' if strip_prompt_strings else self.formatter.prompt
else:
it = self.prompt_frame.begin()
while not it.atEnd():
bl = it.currentBlock()
t = unicode(bl.text())
if strip_prompt_strings:
t = t[self.prompt_len:]
yield t
it += 1
def set_prompt(self, lines):
self.render_current_prompt(lines)
def clear_current_prompt(self):
if self.prompt_frame is None:
c = self.root_frame.lastCursorPosition()
self.prompt_frame = c.insertFrame(self.prompt_frame_format)
self.setTextCursor(c)
else:
c = self.prompt_frame.firstCursorPosition()
self.setTextCursor(c)
c.setPosition(self.prompt_frame.lastPosition(), c.KeepAnchor)
c.removeSelectedText()
c.setPosition(self.prompt_frame.firstPosition())
def render_current_prompt(self, lines=None, restore_cursor=False):
row, col = self.cursor_pos
cp = list(self.prompt()) if lines is None else lines
self.clear_current_prompt()
for i, line in enumerate(cp):
start = i == 0
end = i == len(cp) - 1
self.formatter.render_prompt(not start, self.cursor)
self.formatter.render(self.lexer.get_tokens(line), self.cursor)
if not end:
self.cursor.insertBlock()
if row > -1 and restore_cursor:
self.cursor_pos = (row, col)
self.ensureCursorVisible()
# }}}
# Non-prompt Rendering {{{
def render_block(self, text, restore_prompt=True):
self.formatter.render(self.lexer.get_tokens(text), self.cursor)
self.cursor.insertBlock()
self.cursor.movePosition(self.cursor.End)
if restore_prompt:
self.render_current_prompt()
def show_error(self, is_syntax_err, tb, controller=None):
if self.prompt_frame is not None:
# At a prompt, so redirect output
return prints(tb, end='')
try:
self.buf.append(tb)
if is_syntax_err:
self.formatter.render_syntax_error(tb, self.cursor)
else:
self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
except:
prints(tb, end='')
self.ensureCursorVisible()
QApplication.processEvents()
def show_output(self, raw, which='stdout', controller=None):
def do_show():
try:
self.buf.append(raw)
self.formatter.render_raw(raw, self.cursor)
except:
import traceback
prints(traceback.format_exc())
prints(raw, end='')
if self.prompt_frame is not None:
with Prepender(self):
do_show()
else:
do_show()
self.ensureCursorVisible()
QApplication.processEvents()
# }}}
# Keyboard management {{{
def keyPressEvent(self, ev):
text = unicode(ev.text())
key = ev.key()
action = self.key_dispatcher.get(key, None)
if callable(action):
action()
elif key in (Qt.Key_Escape,):
QTextEdit.keyPressEvent(self, ev)
elif text:
self.text_typed(text)
else:
QTextEdit.keyPressEvent(self, ev)
def left_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if pos > self.prompt_len:
c = self.cursor
c.movePosition(c.PreviousCharacter)
self.setTextCursor(c)
elif lineno > 0:
c = self.cursor
c.movePosition(c.Up)
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def up_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if lineno == 0:
b = self.history.back()
if b is not None:
self.set_prompt(b)
else:
c = self.cursor
c.movePosition(c.Up)
self.setTextCursor(c)
self.ensureCursorVisible()
def backspace_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
if pos > self.prompt_len:
self.cursor.deletePreviousChar()
elif lineno > 0:
c = self.cursor
c.movePosition(c.Up)
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def delete_pressed(self):
self.cursor.deleteChar()
self.ensureCursorVisible()
def right_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
c = self.cursor
cp = list(self.prompt(False))
if pos < len(cp[lineno]):
c.movePosition(c.NextCharacter)
elif lineno < len(cp)-1:
c.movePosition(c.NextCharacter, n=1+self.prompt_len)
self.setTextCursor(c)
self.ensureCursorVisible()
def down_pressed(self):
lineno, pos = self.cursor_pos
if lineno < 0: return
c = self.cursor
cp = list(self.prompt(False))
if lineno >= len(cp) - 1:
b = self.history.forward()
if b is not None:
self.set_prompt(b)
else:
c = self.cursor
c.movePosition(c.Down)
self.setTextCursor(c)
self.ensureCursorVisible()
def home_pressed(self):
if self.prompt_frame is not None:
mods = QApplication.keyboardModifiers()
ctrl = bool(int(mods & Qt.CTRL))
if ctrl:
self.cursor_pos = (0, self.prompt_len)
else:
c = self.cursor
c.movePosition(c.StartOfLine)
c.movePosition(c.NextCharacter, n=self.prompt_len)
self.setTextCursor(c)
self.ensureCursorVisible()
def end_pressed(self):
if self.prompt_frame is not None:
mods = QApplication.keyboardModifiers()
ctrl = bool(int(mods & Qt.CTRL))
if ctrl:
self.cursor_pos = (len(list(self.prompt()))-1, self.prompt_len)
c = self.cursor
c.movePosition(c.EndOfLine)
self.setTextCursor(c)
self.ensureCursorVisible()
def enter_pressed(self):
if self.prompt_frame is None:
return
if not self.controller.is_alive:
return self.no_controller_error()
cp = list(self.prompt())
if cp[0]:
try:
ret = self.compiler('\n'.join(cp))
except:
pass
else:
if ret is None:
c = self.prompt_frame.lastCursorPosition()
c.insertBlock()
self.setTextCursor(c)
self.render_current_prompt()
return
else:
self.history.enter(cp)
self.execute(cp)
def text_typed(self, text):
if self.prompt_frame is not None:
self.move_cursor_to_prompt()
self.cursor.insertText(text)
self.render_current_prompt(restore_cursor=True)
self.history.current = list(self.prompt())