Exemplo n.º 1
0
class PythonTest(unittest.TestCase):
    def setUp(self):
        self.lexer = PythonLexer()

    def test_cls_builtin(self):
        """
        Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo

        """
        fragment = 'class TestClass():\n    @classmethod\n    def hello(cls):\n        pass\n'
        tokens = [
            (Token.Keyword, 'class'),
            (Token.Text, ' '),
            (Token.Name.Class, 'TestClass'),
            (Token.Punctuation, '('),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Name.Decorator, '@classmethod'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Keyword, 'def'),
            (Token.Text, ' '),
            (Token.Name.Function, 'hello'),
            (Token.Punctuation, '('),
            (Token.Name.Builtin.Pseudo, 'cls'),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '        '),
            (Token.Keyword, 'pass'),
            (Token.Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class PygmentsHighlighter(object):
    """ highlight python code with a QSyntaxHighlighter,
        callable class (e.g. function with a state) to """

    def __init__(self):
        """ constructor """
        self._lexer = PythonLexer()
        self._formatter = HtmlFormatter()
        self._document = QtGui.QTextDocument()
        self._document.setDefaultStyleSheet(self._formatter.get_style_defs())
        self._format_cache = dict()

    def __call__(self, code):
        """ makes this class callable, actually do the highlightning """
        index = 0
        for token, text in self._lexer.get_tokens(code):
            length = len(text)
            char_format = self._get_format(token)
            pygmentsHighlighter._setFormat(index, length, char_format)
            index += length

    def _get_format(self, token):
        """ get the QTextCharFormat for a token """
        if token in self._format_cache:
            return self._format_cache[token]

        # get format from document
        code, html = next(self._formatter._format_lines([(token, u"dummy")]))
        self._document.setHtml(html)
        char_format = QtGui.QTextCursor(self._document).charFormat()

        # cache result
        self._format_cache[token] = char_format

        return char_format
Exemplo n.º 3
0
class PygmentsHighlighter(object):
    """ highlight python code with a QSyntaxHighlighter,
        callable class (e.g. function with a state) to """
    def __init__(self):
        """ constructor """
        self._lexer = PythonLexer()
        self._formatter = HtmlFormatter()
        self._document = QtGui.QTextDocument()
        self._document.setDefaultStyleSheet(self._formatter.get_style_defs())
        self._format_cache = dict()

    def __call__(self, code):
        """ makes this class callable, actually do the highlightning """
        index = 0
        for token, text in self._lexer.get_tokens(code):
            length = len(text)
            char_format = self._get_format(token)
            pygmentsHighlighter._setFormat(index, length, char_format)
            index += length

    def _get_format(self, token):
        """ get the QTextCharFormat for a token """
        if token in self._format_cache:
            return self._format_cache[token]

        # get format from document
        code, html = self._formatter._format_lines([(token, u'dummy')]).next()
        self._document.setHtml(html)
        char_format = QtGui.QTextCursor(self._document).charFormat()

        # cache result
        self._format_cache[token] = char_format

        return char_format
Exemplo n.º 4
0
def python_line_tokens(code_lines, blank_lines=False):
    from pygments.lexers import PythonLexer
    lexer = PythonLexer()
    code_str = "".join(code_lines)
    all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
    line_tokens = []
    current_line = []

    for t in all_tokens:
        if t[1] == u"\n":
            line_tokens.append(current_line)
            current_line = []
        else:
            current_line.append(t)

    rows = []
    for i, tokens in enumerate(line_tokens):
        # Check for blank line
        line_str = code_lines[i].rstrip()
        if (not blank_lines) and len(line_str.strip()) == 0:
            continue

        for t in tokens:
            kind, value = str(t[0]), t[1]
            yield line_str, i, kind, value, t
Exemplo n.º 5
0
class PythonTest(unittest.TestCase):
    def setUp(self):
        self.lexer = PythonLexer()

    def test_cls_builtin(self):
        """
        Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo

        """
        fragment = 'class TestClass():\n    @classmethod\n    def hello(cls):\n        pass\n'
        tokens = [
            (Token.Keyword, 'class'),
            (Token.Text, ' '),
            (Token.Name.Class, 'TestClass'),
            (Token.Punctuation, '('),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Name.Decorator, '@classmethod'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Keyword, 'def'),
            (Token.Text, ' '),
            (Token.Name.Function, 'hello'),
            (Token.Punctuation, '('),
            (Token.Name.Builtin.Pseudo, 'cls'),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '        '),
            (Token.Keyword, 'pass'),
            (Token.Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
Exemplo n.º 6
0
def python_token_metrics(code_lines, indent_size=4):
    from pygments.lexers import PythonLexer
    indent_regex = re.compile(r"^\s*")

    lexer = PythonLexer()
    code_str = "".join(code_lines)
    all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
    line_tokens = []
    current_line = []

    for t in all_tokens:
        if t[1] == u"\n":
            line_tokens.append(current_line)
            current_line = []
        else:
            current_line.append(t)

    rows = []
    for i, tokens in enumerate(line_tokens):
        line_number = i + 1

        # Check for blank line
        line_str = code_lines[i].rstrip()
        if len(line_str.strip()) == 0:
            rows.append([line_number, 0, 0, 0, 0, 0, 0])
            continue

        assert len(tokens) > 0, "No tokens for line"

        num_keywords = 0
        num_identifiers = 0
        num_operators = 0
        line_length = len(line_str)
        line_indent = len(indent_regex.findall(line_str)[0]) / indent_size

        # Indentation is not considered
        line_str_noindent = line_str.lstrip()
        line_length_noindent = len(line_str_noindent)
        whitespace_prop = line_str_noindent.count(" ") / float(line_length_noindent)

        for t in tokens:
            kind, value = str(t[0]), t[1]
            if kind.startswith(u"Token.Keyword"):
                num_keywords += 1
            elif kind.startswith(u"Token.Name"):
                num_identifiers += 1
            elif kind.startswith(u"Token.Operator"):
                num_operators += 1

        rows.append([line_number, line_length_noindent, num_keywords,
            num_identifiers, num_operators, whitespace_prop,
            line_indent])

    columns = ["line", "line_length", "keywords",
               "identifiers", "operators", "whitespace_prop",
               "line_indent"]
    return pandas.DataFrame(rows, columns=columns)
Exemplo n.º 7
0
    def _analyse_source_code(self, source_code):

        lexer = PythonLexer()
        token_source = lexer.get_tokens(source_code)
        for token_type, value in token_source:
            if len(value) > 3 and value.startswith('gl') and ord('A') <= ord(value[2]) <= ord('Z'):
                self.gl_functions.add(value)
            elif len(value) > 3 and value.startswith('GL_'):
                self.gl_constants.add(value)
Exemplo n.º 8
0
class CursesParser:
    def __init__(self, makecolors = True, style = standardcols):
        if makecolors:
            self.makecolorpairs()
        if style is None:
            style = standardcols
        self.style = style
        self.lexer = PythonLexer()
    
    @classmethod
    def makecolorpairs(cls):
        """Initializes curses for colors, makes a color pair of 
        (color, defaultbg) for every color, and initializes self.colorpairs as
        a dictionary with a color -> colorpair mapping"""
        if hasattr(cls, 'colorpairs'):
            return cls.colorpairs
        curses.start_color()
        curses.use_default_colors()
        maxcolors = curses.COLORS
        maxpairs = curses.COLOR_PAIRS
        totalmax = min(maxcolors+1, maxpairs)
        cls.colorpairs = {}
        for colpr in range(1,totalmax):
            if colpr >= maxpairs:
                break
            col = colpr % maxcolors
            curses.init_pair(colpr, col, -1)
            cls.colorpairs[col] = curses.color_pair(colpr)
        return cls.colorpairs
    
    def get_colors(self, raw):
        """Uses pygments to parse the text, and yields (text, color, attr)
        tuples"""
        for tkn, txt in self.lexer.get_tokens(raw):
            notyielded = True
            while notyielded:
                if tkn is None:
                    yield (txt, None, None)
                    notyielded = False
                elif tkn in self.style:
                    col, attr = self.style[tkn]
                    yield (txt, col, attr)
                    notyielded = False
                else:
                    tkn = tkn.parent
    
    def parsetoscr(self, scr, raw):
        """Parses text, and uses scr.addstr to print the text directly."""
        self.makecolorpairs()
        for (txt, col, attr) in self.get_colors(raw):
            fullattr = attr
            if attr is None:
                fullattr = curses.A_NORMAL
            if col is not None: # and col in self.colorpairs:
                fullattr |= self.colorpairs[col]
            scr.addstr(txt, fullattr)
Exemplo n.º 9
0
    def testWorksAsExpected(self):
        code = '''
        """ Increment number of decision points in function."""
        #if tok and tok.text in McCabeKeywords:
        if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
            self.metrics['mccabe'] += 1
        '''
        result = [(Token.Text, u'        '), (Token.Literal.String.Doc, u'""" Increment number of decision points in function."""'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Comment, u'#if tok and tok.text in McCabeKeywords:'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Keyword, u'if'), (Token.Text, u' '), (Token.Punctuation, u'('), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'=='), (Token.Text, u' '), (Token.Name, u'b'), (Token.Literal.String, u"'"), (Token.Literal.String, u'Keyword'), (Token.Literal.String, u"'"), (Token.Punctuation, u')'), (Token.Text, u' '), (Token.Operator.Word, u'and'), (Token.Text, u' '), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'1'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator.Word, u'in'), (Token.Text, u' '), (Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'), (Token.Text, u'\n'), (Token.Text, u'            '), (Token.Name.Builtin.Pseudo, u'self'), (Token.Operator, u'.'), (Token.Name, u'metrics'), (Token.Punctuation, u'['), (Token.Literal.String, u"'"), (Token.Literal.String, u'mccabe'), (Token.Literal.String, u"'"), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'+'), (Token.Operator, u'='), (Token.Text, u' '), (Token.Literal.Number.Integer, u'1'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Text, u'\n')]

        lex = PythonLexer()
        tokenList = lex.get_tokens(code)
        self.assertEqual(list(tokenList), result)
Exemplo n.º 10
0
    def testWorksAsExpected(self):
        code = '''
        """ Increment number of decision points in function."""
        #if tok and tok.text in McCabeKeywords:
        if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
            self.metrics['mccabe'] += 1
        '''
        result = [(Token.Text, u'        '),
                  (Token.Literal.String.Doc,
                   u'""" Increment number of decision points in function."""'),
                  (Token.Text, u'\n'), (Token.Text, u'        '),
                  (Token.Comment, u'#if tok and tok.text in McCabeKeywords:'),
                  (Token.Text, u'\n'), (Token.Text, u'        '),
                  (Token.Keyword, u'if'), (Token.Text, u' '),
                  (Token.Punctuation, u'('), (Token.Name, u'tok'),
                  (Token.Punctuation, u'['),
                  (Token.Literal.Number.Integer, u'0'),
                  (Token.Punctuation, u']'), (Token.Punctuation, u'['),
                  (Token.Literal.Number.Integer, u'0'),
                  (Token.Punctuation, u']'), (Token.Text, u' '),
                  (Token.Operator, u'=='), (Token.Text, u' '),
                  (Token.Name, u'b'), (Token.Literal.String, u"'"),
                  (Token.Literal.String, u'Keyword'),
                  (Token.Literal.String, u"'"), (Token.Punctuation, u')'),
                  (Token.Text, u' '), (Token.Operator.Word, u'and'),
                  (Token.Text, u' '), (Token.Name, u'tok'),
                  (Token.Punctuation, u'['),
                  (Token.Literal.Number.Integer, u'1'),
                  (Token.Punctuation, u']'), (Token.Text, u' '),
                  (Token.Operator.Word, u'in'), (Token.Text, u' '),
                  (Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'),
                  (Token.Text, u'\n'), (Token.Text, u'            '),
                  (Token.Name.Builtin.Pseudo, u'self'), (Token.Operator, u'.'),
                  (Token.Name, u'metrics'), (Token.Punctuation, u'['),
                  (Token.Literal.String, u"'"),
                  (Token.Literal.String, u'mccabe'),
                  (Token.Literal.String, u"'"), (Token.Punctuation, u']'),
                  (Token.Text, u' '), (Token.Operator, u'+'),
                  (Token.Operator, u'='), (Token.Text, u' '),
                  (Token.Literal.Number.Integer, u'1'), (Token.Text, u'\n'),
                  (Token.Text, u'        '), (Token.Text, u'\n')]

        lex = PythonLexer()
        tokenList = lex.get_tokens(code)
        self.assertEqual(list(tokenList), result)
Exemplo n.º 11
0
def get_context(string):
    """ Assuming the cursor is at the end of the specified string, get the
        context (a list of names) for the symbol at cursor position.
    """
    lexer = PythonLexer()
    context = []
    reversed_tokens = list(lexer.get_tokens(string))
    reversed_tokens.reverse()

    # Pygments often tacks on a newline when none is specified in the input.
    # Remove this newline.
    if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
            not string.endswith('\n'):
        reversed_tokens.pop(0)

    current_op = ''
    for token, text in reversed_tokens:

        if is_token_subtype(token, Token.Name):

            # Handle a trailing separator, e.g 'foo.bar.'
            if current_op == '.':
                if not context:
                    context.insert(0, '')

            # Handle non-separator operators and punction.
            elif current_op:
                break

            context.insert(0, text)
            current_op = ''

        # Pygments doesn't understand that, e.g., '->' is a single operator
        # in C++. This is why we have to build up an operator from
        # potentially several tokens.
        elif token is Token.Operator or token is Token.Punctuation:
            current_op = text + current_op

        # Break on anything that is not a Operator, Punctuation, or Name.
        else:
            break

    return context
Exemplo n.º 12
0
def get_context(string):
    """ Assuming the cursor is at the end of the specified string, get the
        context (a list of names) for the symbol at cursor position.
    """
    lexer = PythonLexer()
    context = []
    reversed_tokens = list(lexer.get_tokens(string))
    reversed_tokens.reverse()

    # Pygments often tacks on a newline when none is specified in the input.
    # Remove this newline.
    if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
            not string.endswith('\n'):
        reversed_tokens.pop(0)

    current_op = ''
    for token, text in reversed_tokens:

        if is_token_subtype(token, Token.Name):

            # Handle a trailing separator, e.g 'foo.bar.'
            if current_op == '.':
                if not context:
                    context.insert(0, '')

            # Handle non-separator operators and punction.
            elif current_op:
                break

            context.insert(0, text)
            current_op = ''

        # Pygments doesn't understand that, e.g., '->' is a single operator
        # in C++. This is why we have to build up an operator from
        # potentially several tokens.
        elif token is Token.Operator or token is Token.Punctuation:
            current_op = text + current_op

        # Break on anything that is not a Operator, Punctuation, or Name.
        else:
            break

    return context
Exemplo n.º 13
0
def test_lexer_on_python_global_code():
    code = textwrap.dedent('''
        """ Increment number of decision points in function."""
        #if tok and tok.text in McCabeKeywords:
        if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
        self.metrics['mccabe'] += 1
        ''')
    result = [
        (Token.Literal.String.Doc,
         u'""" Increment number of decision points in function."""'),
        (Token.Text, u'\n'),
        (Token.Comment.Single, u'#if tok and tok.text in McCabeKeywords:'),
        (Token.Text, u'\n'), (Token.Keyword, u'if'), (Token.Text, u' '),
        (Token.Punctuation, u'('), (Token.Name, u'tok'),
        (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'),
        (Token.Punctuation, u']'), (Token.Punctuation, u'['),
        (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'),
        (Token.Text, u' '), (Token.Operator, u'=='), (Token.Text, u' '),
        (Token.Literal.String.Affix, u'b'),
        (Token.Literal.String.Single, u"'"),
        (Token.Literal.String.Single, u'Keyword'),
        (Token.Literal.String.Single, u"'"), (Token.Punctuation, u')'),
        (Token.Text, u' '), (Token.Operator.Word, u'and'), (Token.Text, u' '),
        (Token.Name, u'tok'), (Token.Punctuation, u'['),
        (Token.Literal.Number.Integer, u'1'), (Token.Punctuation, u']'),
        (Token.Text, u' '), (Token.Operator.Word, u'in'), (Token.Text, u' '),
        (Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'),
        (Token.Text, u'\n'), (Token.Name.Builtin.Pseudo, u'self'),
        (Token.Operator, u'.'), (Token.Name, u'metrics'),
        (Token.Punctuation, u'['), (Token.Literal.String.Single, u"'"),
        (Token.Literal.String.Single, u'mccabe'),
        (Token.Literal.String.Single, u"'"), (Token.Punctuation, u']'),
        (Token.Text, u' '), (Token.Operator, u'+'), (Token.Operator, u'='),
        (Token.Text, u' '), (Token.Literal.Number.Integer, u'1'),
        (Token.Text, u'\n')
    ]

    lex = PythonLexer()
    tokenList = lex.get_tokens(code)
    # print(list(tokenList))
    assert list(tokenList) == result
Exemplo n.º 14
0
def test_lexer_on_python_function():
    code = textwrap.dedent('''
        def my_func(p1, p2):
            """Work the incredible magic."""
            return p1 + p2
        ''')
    result = [(Token.Keyword, u'def'), (Token.Text, u' '),
              (Token.Name.Function, u'my_func'), (Token.Punctuation, u'('),
              (Token.Name, u'p1'), (Token.Punctuation, u','),
              (Token.Text, u' '), (Token.Name, u'p2'),
              (Token.Punctuation, u')'), (Token.Punctuation, u':'),
              (Token.Text, u'\n'), (Token.Text, u'    '),
              (Token.Literal.String.Doc, u'"""Work the incredible magic."""'),
              (Token.Text, u'\n'), (Token.Text, u'    '),
              (Token.Keyword, u'return'), (Token.Text, u' '),
              (Token.Name, u'p1'), (Token.Text, u' '), (Token.Operator, u'+'),
              (Token.Text, u' '), (Token.Name, u'p2'), (Token.Text, u'\n')]

    lex = PythonLexer()
    tokenList = lex.get_tokens(code)
    # print(list(tokenList))
    assert list(tokenList) == result
Exemplo n.º 15
0
def test_lexer_on_python_class():
    code = textwrap.dedent('''
        class MyClass:
            """Definitely first class!"""
            variable = "blah"

            def my_func(self):
                """Work the incredible magic."""
                print("This is a message.")
        ''')
    result = [(Token.Keyword, u'class'), (Token.Text, u' '),
              (Token.Name.Class, u'MyClass'), (Token.Punctuation, u':'),
              (Token.Text, u'\n'), (Token.Text, u'    '),
              (Token.Literal.String.Doc, u'"""Definitely first class!"""'),
              (Token.Text, u'\n'), (Token.Text, u'    '),
              (Token.Name, u'variable'), (Token.Text, u' '),
              (Token.Operator, u'='), (Token.Text, u' '),
              (Token.Literal.String.Double, u'"'),
              (Token.Literal.String.Double, u'blah'),
              (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'),
              (Token.Text, u'\n'), (Token.Text, u'    '),
              (Token.Keyword, u'def'), (Token.Text, u' '),
              (Token.Name.Function, u'my_func'), (Token.Punctuation, u'('),
              (Token.Name.Builtin.Pseudo, u'self'), (Token.Punctuation, u')'),
              (Token.Punctuation, u':'), (Token.Text, u'\n'),
              (Token.Text, u'        '),
              (Token.Literal.String.Doc, u'"""Work the incredible magic."""'),
              (Token.Text, u'\n'), (Token.Text, u'        '),
              (Token.Keyword, u'print'), (Token.Punctuation, u'('),
              (Token.Literal.String.Double, u'"'),
              (Token.Literal.String.Double, u'This is a message.'),
              (Token.Literal.String.Double, u'"'), (Token.Punctuation, u')'),
              (Token.Text, u'\n')]

    lex = PythonLexer()
    tokenList = lex.get_tokens(code)
    # print(list(tokenList))
    assert list(tokenList) == result
Exemplo n.º 16
0
    def __tokenizeFromText(self, text):
        lexer = PythonLexer()  # Using pygments Python Lexer
        tokens = lexer.get_tokens(text)
        tokens = list(tokens)  # Convert to tokens to list object
        result = []
        prev_c = ''  # Remember previous category
        row = 0  # Remember position of row
        col = 0  # Remember position of column

        # Simplify tokens using categories and add additional coordinates
        for token in tokens:
            c = get_category(token)

            if (c != None):
                # Linefeed detected -> do not append to result but change position
                if c == 'L':
                    row = row + 1  #Increment line position
                    col = 0  # Set back column after linefeed

                # New block detected
                elif prev_c == 'L' and c != 'I' and result:
                    self.blocks.append(Block(result))
                    result = []

                if c != 'L':
                    result.append((c, row, col,
                                   token[1]))  # Append result for single token
                    col += 1  #Increment column position
                    if col > self._max_col:
                        self._max_col = col
            prev_c = c
        self._max_row = row  # Set max row for Tokens instance

        # Append last block if result not empty
        if result:
            self.blocks.append(Block(result))
Exemplo n.º 17
0
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):

    """ Syntax highlighter that uses Pygments for parsing. """

    style = DefaultStyle  # Could be MonokaiStyle, FriendlyStyle, etc.

    def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        try:
            self._lexer = get_lexer_by_name(lexer)
        except:
            self._lexer = PythonLexer()

        # Caches for formats and brushes.
        self._brushes = {}
        self._formats = {}

    @handle_exception_in_method
    def highlightBlock(self, qstring):
        """ Highlight a block of text.
        """
        qstring = compat.unicode(qstring)
        prev_data = self.previous_block_data()

        if prev_data is not None:
            self._lexer._epd_state_stack = prev_data.syntax_stack
        elif hasattr(self._lexer, '_epd_state_stack'):
            del self._lexer._epd_state_stack

        index = 0
        # Lex the text using Pygments
        for token, text in self._lexer.get_tokens(qstring):
            l = len(text)
            format = self._get_format(token)
            if format is not None:
                self.setFormat(index, l, format)
            index += l

        if hasattr(self._lexer, '_epd_state_stack'):
            data = BlockUserData(syntax_stack=self._lexer._epd_state_stack)
            self.currentBlock().setUserData(data)

            # there is a bug in pyside and it will crash unless we
            # hold on to the reference a little longer
            data = self.currentBlock().userData()

            # Clean up for the next go-round.
            del self._lexer._epd_state_stack

    def previous_block_data(self):
        """ Convenience method for returning the previous block's user data.
        """
        return self.currentBlock().previous().userData()

    def _get_format(self, token):
        """ Returns a QTextCharFormat for token or None.
        """
        if token in self._formats:
            return self._formats[token]
        result = None
        for key, value in self.style.style_for_token(token) .items():
            if value:
                if result is None:
                    result = QtGui.QTextCharFormat()
                if key == 'color':
                    result.setForeground(self._get_brush(value))
                elif key == 'bgcolor':
                    result.setBackground(self._get_brush(value))
                elif key == 'bold':
                    result.setFontWeight(QtGui.QFont.Bold)
                elif key == 'italic':
                    result.setFontItalic(True)
                elif key == 'underline':
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.SingleUnderline)
                elif key == 'sans':
                    result.setFontStyleHint(QtGui.QFont.SansSerif)
                elif key == 'roman':
                    result.setFontStyleHint(QtGui.QFont.Times)
                elif key == 'mono':
                    result.setFontStyleHint(QtGui.QFont.TypeWriter)
                elif key == 'border':
                    # Borders are normally used for errors. We can't do a border
                    # so instead we do a wavy underline
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.WaveUnderline)
                    result.setUnderlineColor(self._get_color(value))
        self._formats[token] = result
        return result

    def _get_brush(self, color):
        """ Returns a brush for the color.
        """
        result = self._brushes.get(color)
        if result is None:
            qcolor = self._get_color(color)
            result = QtGui.QBrush(qcolor)
            self._brushes[color] = result

        return result

    def _get_color(self, color):
        qcolor = QtGui.QColor()
        qcolor.setRgb(int(color[:2], base=16),
                      int(color[2:4], base=16),
                      int(color[4:6], base=16))
        return qcolor
Exemplo n.º 18
0
def _lex_python_result(tb):
    " Return token list for Python string. "
    lexer = PythonLexer()
    return lexer.get_tokens(tb)
Exemplo n.º 19
0
def make_code_aois(code_file,
                   font_size=(14, 25),
                   line_offset=5,
                   syntax_categories=SYNTAX_CATEGORIES):

    aoi_df = pandas.DataFrame(columns=("aoi_kind", "name", "x", "y", "width",
                                       "height", "note"))

    # Needed for syntax-based AOIs
    from pygments.lexers import PythonLexer
    lexer = PythonLexer()

    # Parse each file and generate AOIs
    code_lines = code_file.readlines()
    code_str = "".join(code_lines)

    # Add extra newline token to trigger last AOI block
    code_lines += [""]
    tokens = list(lexer.get_tokens(code_str,
                                   unfiltered=True)) + [("Token.Text", u"\n")]

    col = 0  # Current column
    line = 0  # Current line number
    block_start = 0  # Current whitespace separated block
    last_blank = False  # Was last line blank?
    block_lines = []  # Lines in current block

    for t in tokens:
        kind = str(t[0])
        val = t[1]

        # Check if end of line
        if val == u"\n":
            line_str = code_lines[line].rstrip()
            if len(line_str.strip()) > 0:
                # Non-blank line: add AOI for whole line
                aoi_df = aoi_df.append(
                    {
                        "aoi_kind":
                        "line",
                        "name":
                        "line {0}".format(line + 1),
                        "x":
                        0,
                        "y": (line * font_size[1]) + (line * line_offset) -
                        (line_offset / 2),
                        "width":
                        len(line_str) * font_size[0],
                        "height":
                        font_size[1] + line_offset - 1,
                        "note":
                        line_str
                    },
                    ignore_index=True)

                # Add to current block
                last_blank = False
                block_lines.append(line_str)
            else:
                # Blank line
                if not last_blank:
                    # Add AOI for whitespace separated block of lines
                    aoi_df = aoi_df.append(
                        {
                            "aoi_kind":
                            "block",
                            "name":
                            "lines {0}-{1}".format(block_start + 1, line + 1),
                            "x":
                            0,
                            "y": (block_start * font_size[1]) +
                            (block_start * line_offset) - (line_offset / 2),
                            "width":
                            max([len(l) for l in block_lines]) * font_size[0],
                            "height":
                            len(block_lines) * (font_size[1] + line_offset),
                            "note":
                            "\n".join(block_lines)
                        },
                        ignore_index=True)

                # Reset block variables
                last_blank = True
                block_lines = []
                block_start = line + 1

            # Next line
            col = 0
            line += 1
            continue

        # Add AOI for syntax token
        if (kind == "Token.Text") and (col == 0):
            kind += ".Indentation"

        aoi_df = aoi_df.append(
            {
                "aoi_kind":
                "syntax",
                "name":
                syntax_categories[kind],
                "x":
                col * font_size[0],
                "y": (line * font_size[1]) + (line * line_offset) -
                (line_offset / 2),
                "width":
                len(val) * font_size[0],
                "height":
                font_size[1] + line_offset - 1,
                "note":
                val
            },
            ignore_index=True)

        col += len(val)

    return aoi_df
Exemplo n.º 20
0
def python_line_categories(code_lines):
    from pygments.lexers import PythonLexer

    lexer = PythonLexer()
    code_str = "".join(code_lines)
    all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
    line_tokens = []
    current_line = []

    for t in all_tokens:
        if t[1] == u"\n":
            line_tokens.append(current_line)
            current_line = []
        else:
            current_line.append(t)

    line_categories = []
    for i, tokens in enumerate(line_tokens):
        # Check for blank line
        line_str = code_lines[i].rstrip()
        if len(line_str.strip()) == 0:
            line_categories.append(["blank line"])
            continue

        assert len(tokens) > 0, "No tokens for line"
        categories = []
        last_kind, last_value = None, None

        for t in tokens:
            kind, value = str(t[0]), t[1]

            if kind == u"Token.Keyword" and value == u"def":
                categories.append("function definition")
            elif kind == u"Token.Keyword" and value == u"if":
                categories.append("if statement")
            elif kind == u"Token.Keyword" and value == u"for":
                categories.append("for loop")
            elif kind == u"Token.Keyword" and value == u"return":
                categories.append("return statement")
            elif kind == u"Token.Keyword" and value == u"print":
                categories.append("print statement")
            elif kind == u"Token.Keyword" and value == u"class":
                categories.append("class definition")
            elif kind == u"Token.Operator" and value == u"=":
                categories.append("assignment")
            elif kind == u"Token.Operator" and value == u".":
                categories.append("object access")
            elif kind == u"Token.Operator" and value in [u"+", u"*"]:
                categories.append("mathematical operation")
            elif last_kind == u"Token.Operator" and last_value == u"-" and kind == "Token.Whitespace":
                categories.append("mathematical operation")
            elif kind == u"Token.Operator" and value in [u"<", u">"]:
                categories.append("comparison")
            elif last_kind == u"Token.Name" and kind == "Token.Punctuation" and value == u"(":
                categories.append("function call")
            elif kind == "Token.Punctuation" and value == u"[":
                categories.append("list creation")

            last_kind, last_value = kind, value

        if len(categories) == 0:
            categories.append("unknown")

        line_categories.append(set(categories))

    return line_categories
Exemplo n.º 21
0
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
    """ Syntax highlighter that uses Pygments for parsing. """
    def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        try:
            self._lexer = get_lexer_by_name(lexer)
        except:
            self._lexer = PythonLexer()

        self._style = DefaultStyle
        # Caches for formats and brushes.
        self._brushes = {}
        self._formats = {}

    def highlightBlock(self, qstring):
        """ Highlight a block of text.
        """
        qstring = str(qstring)
        prev_data = self.previous_block_data()

        if prev_data is not None:
            self._lexer._saved_state_stack = prev_data.syntax_stack
        elif hasattr(self._lexer, "_saved_state_stack"):
            del self._lexer._saved_state_stack

        index = 0
        # Lex the text using Pygments
        for token, text in self._lexer.get_tokens(qstring):
            l = len(text)
            format = self._get_format(token)
            if format is not None:
                self.setFormat(index, l, format)
            index += l

        if hasattr(self._lexer, "_saved_state_stack"):
            data = BlockUserData(syntax_stack=self._lexer._saved_state_stack)
            self.currentBlock().setUserData(data)

            # there is a bug in pyside and it will crash unless we
            # hold on to the reference a little longer
            data = self.currentBlock().userData()

            # Clean up for the next go-round.
            del self._lexer._saved_state_stack

    def previous_block_data(self):
        """ Convenience method for returning the previous block's user data.
        """
        return self.currentBlock().previous().userData()

    def _get_format(self, token):
        """ Returns a QTextCharFormat for token or None.
        """
        if token in self._formats:
            return self._formats[token]
        result = None
        while not self._style.styles_token(token):
            token = token.parent
        for key, value in self._style.style_for_token(token).items():
            if value:
                if result is None:
                    result = QtGui.QTextCharFormat()
                if key == "color":
                    result.setForeground(self._get_brush(value))
                elif key == "bgcolor":
                    result.setBackground(self._get_brush(value))
                elif key == "bold":
                    result.setFontWeight(QtGui.QFont.Bold)
                elif key == "italic":
                    result.setFontItalic(True)
                elif key == "underline":
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.SingleUnderline)
                elif key == "sans":
                    result.setFontStyleHint(QtGui.QFont.SansSerif)
                elif key == "roman":
                    result.setFontStyleHint(QtGui.QFont.Times)
                elif key == "mono":
                    result.setFontStyleHint(QtGui.QFont.TypeWriter)
                elif key == "border":
                    # Borders are normally used for errors. We can't do a border
                    # so instead we do a wavy underline
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.WaveUnderline)
                    result.setUnderlineColor(self._get_color(value))
        self._formats[token] = result
        return result

    def _get_brush(self, color):
        """ Returns a brush for the color.
        """
        result = self._brushes.get(color)
        if result is None:
            qcolor = self._get_color(color)
            result = QtGui.QBrush(qcolor)
            self._brushes[color] = result

        return result

    def _get_color(self, color):
        qcolor = QtGui.QColor()
        qcolor.setRgb(
            int(color[:2], base=16),
            int(color[2:4], base=16),
            int(color[4:6], base=16),
        )
        return qcolor
Exemplo n.º 22
0
def make_code_aois(code_file, font_size=(14, 25), line_offset=5,
        syntax_categories=SYNTAX_CATEGORIES):

    aoi_df = pandas.DataFrame(columns=("aoi_kind", "name",
        "x", "y", "width", "height", "note"))

    # Needed for syntax-based AOIs
    from pygments.lexers import PythonLexer
    lexer = PythonLexer()

    # Parse each file and generate AOIs
    code_lines = code_file.readlines()
    code_str = "".join(code_lines)

    # Add extra newline token to trigger last AOI block
    code_lines += [""]
    tokens = list(lexer.get_tokens(code_str, unfiltered=True)) + [("Token.Text", u"\n")]

    col = 0            # Current column
    line = 0           # Current line number
    block_start = 0    # Current whitespace separated block
    last_blank = False # Was last line blank?
    block_lines = []   # Lines in current block

    for t in tokens:
        kind = str(t[0])
        val = t[1]

        # Check if end of line
        if val == u"\n":
            line_str = code_lines[line].rstrip()
            if len(line_str.strip()) > 0:
                # Non-blank line: add AOI for whole line
                aoi_df = aoi_df.append({
                    "aoi_kind" : "line",
                    "name"     : "line {0}".format(line + 1),
                    "x"        : 0,
                    "y"        : (line * font_size[1]) + (line * line_offset) - (line_offset / 2),
                    "width"    : len(line_str) * font_size[0],
                    "height"   : font_size[1] + line_offset - 1,
                    "note"     : line_str
                }, ignore_index=True)

                # Add to current block
                last_blank = False
                block_lines.append(line_str)
            else:
                # Blank line
                if not last_blank:
                    # Add AOI for whitespace separated block of lines
                    aoi_df = aoi_df.append({
                        "aoi_kind" : "block",
                        "name"     : "lines {0}-{1}".format(block_start + 1, line + 1),
                        "x"        : 0,
                        "y"        : (block_start * font_size[1]) + (block_start * line_offset) - (line_offset / 2),
                        "width"    : max([len(l) for l in block_lines]) * font_size[0],
                        "height"   : len(block_lines) * (font_size[1] + line_offset),
                        "note"     : "\n".join(block_lines)
                    }, ignore_index=True)

                # Reset block variables
                last_blank = True
                block_lines = []
                block_start = line + 1

            # Next line
            col = 0
            line += 1
            continue

        # Add AOI for syntax token
        if (kind == "Token.Text") and (col == 0):
            kind += ".Indentation"

        aoi_df = aoi_df.append({
            "aoi_kind" : "syntax",
            "name"     : syntax_categories[kind],
            "x"        : col * font_size[0],
            "y"        : (line * font_size[1]) + (line * line_offset) - (line_offset / 2),
            "width"    : len(val) * font_size[0],
            "height"   : font_size[1] + line_offset - 1,
            "note"     : val
        }, ignore_index=True)

        col += len(val)

    return aoi_df
Exemplo n.º 23
0
def _lex_python_result(tb):
    " Return token list for Python string. "
    lexer = PythonLexer()
    return lexer.get_tokens(tb)
Exemplo n.º 24
0
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
    """ Syntax highlighter that uses Pygments for parsing. """

    #---------------------------------------------------------------------------
    # 'QSyntaxHighlighter' interface
    #---------------------------------------------------------------------------

    def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        self._document = self.document()
        self._formatter = HtmlFormatter(nowrap=True)
        self.set_style('default')
        if lexer is not None:
            self._lexer = lexer
        else:
            if PY3:
                self._lexer = Python3Lexer()
            else:
                self._lexer = PythonLexer()

    def highlightBlock(self, string):
        """ Highlight a block of text.
        """
        prev_data = self.currentBlock().previous().userData()
        if prev_data is not None:
            self._lexer._saved_state_stack = prev_data.syntax_stack
        elif hasattr(self._lexer, '_saved_state_stack'):
            del self._lexer._saved_state_stack

        # Lex the text using Pygments
        index = 0
        for token, text in self._lexer.get_tokens(string):
            length = qstring_length(text)
            self.setFormat(index, length, self._get_format(token))
            index += length

        if hasattr(self._lexer, '_saved_state_stack'):
            data = PygmentsBlockUserData(
                syntax_stack=self._lexer._saved_state_stack)
            self.currentBlock().setUserData(data)
            # Clean up for the next go-round.
            del self._lexer._saved_state_stack

    #---------------------------------------------------------------------------
    # 'PygmentsHighlighter' interface
    #---------------------------------------------------------------------------

    def set_style(self, style):
        """ Sets the style to the specified Pygments style.
        """
        if isinstance(style, string_types):
            style = get_style_by_name(style)
        self._style = style
        self._clear_caches()

    def set_style_sheet(self, stylesheet):
        """ Sets a CSS stylesheet. The classes in the stylesheet should
        correspond to those generated by:

            pygmentize -S <style> -f html

        Note that 'set_style' and 'set_style_sheet' completely override each
        other, i.e. they cannot be used in conjunction.
        """
        self._document.setDefaultStyleSheet(stylesheet)
        self._style = None
        self._clear_caches()

    #---------------------------------------------------------------------------
    # Protected interface
    #---------------------------------------------------------------------------

    def _clear_caches(self):
        """ Clear caches for brushes and formats.
        """
        self._brushes = {}
        self._formats = {}

    def _get_format(self, token):
        """ Returns a QTextCharFormat for token or None.
        """
        if token in self._formats:
            return self._formats[token]

        if self._style is None:
            result = self._get_format_from_document(token, self._document)
        else:
            result = self._get_format_from_style(token, self._style)

        self._formats[token] = result
        return result

    def _get_format_from_document(self, token, document):
        """ Returns a QTextCharFormat for token by
        """
        code, html = next(self._formatter._format_lines([(token, u'dummy')]))
        self._document.setHtml(html)
        return QtGui.QTextCursor(self._document).charFormat()

    def _get_format_from_style(self, token, style):
        """ Returns a QTextCharFormat for token by reading a Pygments style.
        """
        result = QtGui.QTextCharFormat()
        for key, value in style.style_for_token(token).items():
            if value:
                if key == 'color':
                    result.setForeground(self._get_brush(value))
                elif key == 'bgcolor':
                    result.setBackground(self._get_brush(value))
                elif key == 'bold':
                    result.setFontWeight(QtGui.QFont.Bold)
                elif key == 'italic':
                    result.setFontItalic(True)
                elif key == 'underline':
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.SingleUnderline)
                elif key == 'sans':
                    result.setFontStyleHint(QtGui.QFont.SansSerif)
                elif key == 'roman':
                    result.setFontStyleHint(QtGui.QFont.Times)
                elif key == 'mono':
                    result.setFontStyleHint(QtGui.QFont.TypeWriter)
        return result

    def _get_brush(self, color):
        """ Returns a brush for the color.
        """
        result = self._brushes.get(color)
        if result is None:
            qcolor = self._get_color(color)
            result = QtGui.QBrush(qcolor)
            self._brushes[color] = result
        return result

    def _get_color(self, color):
        """ Returns a QColor built from a Pygments color string.
        """
        qcolor = QtGui.QColor()
        qcolor.setRgb(int(color[:2], base=16),
                      int(color[2:4], base=16),
                      int(color[4:6], base=16))
        return qcolor
Exemplo n.º 25
0
class PygmentsHighlighter(QtGui.QSyntaxHighlighter):
    """ Syntax highlighter that uses Pygments for parsing. """

    #---------------------------------------------------------------------------
    # 'QSyntaxHighlighter' interface
    #---------------------------------------------------------------------------

    def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        self._document = self.document()
        self._formatter = HtmlFormatter(nowrap=True)
        self.set_style('default')
        if lexer is not None:
            self._lexer = lexer
        else:
            if PY3:
                self._lexer = Python3Lexer()
            else:
                self._lexer = PythonLexer()

    def highlightBlock(self, string):
        """ Highlight a block of text.
        """
        prev_data = self.currentBlock().previous().userData()
        if prev_data is not None:
            self._lexer._saved_state_stack = prev_data.syntax_stack
        elif hasattr(self._lexer, '_saved_state_stack'):
            del self._lexer._saved_state_stack

        # Lex the text using Pygments
        index = 0
        for token, text in self._lexer.get_tokens(string):
            length = len(text)
            self.setFormat(index, length, self._get_format(token))
            index += length

        if hasattr(self._lexer, '_saved_state_stack'):
            data = PygmentsBlockUserData(
                syntax_stack=self._lexer._saved_state_stack)
            self.currentBlock().setUserData(data)
            # Clean up for the next go-round.
            del self._lexer._saved_state_stack

    #---------------------------------------------------------------------------
    # 'PygmentsHighlighter' interface
    #---------------------------------------------------------------------------

    def set_style(self, style):
        """ Sets the style to the specified Pygments style.
        """
        if isinstance(style, string_types):
            style = get_style_by_name(style)
        self._style = style
        self._clear_caches()

    def set_style_sheet(self, stylesheet):
        """ Sets a CSS stylesheet. The classes in the stylesheet should
        correspond to those generated by:

            pygmentize -S <style> -f html

        Note that 'set_style' and 'set_style_sheet' completely override each
        other, i.e. they cannot be used in conjunction.
        """
        self._document.setDefaultStyleSheet(stylesheet)
        self._style = None
        self._clear_caches()

    #---------------------------------------------------------------------------
    # Protected interface
    #---------------------------------------------------------------------------

    def _clear_caches(self):
        """ Clear caches for brushes and formats.
        """
        self._brushes = {}
        self._formats = {}

    def _get_format(self, token):
        """ Returns a QTextCharFormat for token or None.
        """
        if token in self._formats:
            return self._formats[token]

        if self._style is None:
            result = self._get_format_from_document(token, self._document)
        else:
            result = self._get_format_from_style(token, self._style)

        self._formats[token] = result
        return result

    def _get_format_from_document(self, token, document):
        """ Returns a QTextCharFormat for token by
        """
        code, html = next(self._formatter._format_lines([(token, u'dummy')]))
        self._document.setHtml(html)
        return QtGui.QTextCursor(self._document).charFormat()

    def _get_format_from_style(self, token, style):
        """ Returns a QTextCharFormat for token by reading a Pygments style.
        """
        result = QtGui.QTextCharFormat()
        for key, value in style.style_for_token(token).items():
            if value:
                if key == 'color':
                    result.setForeground(self._get_brush(value))
                elif key == 'bgcolor':
                    result.setBackground(self._get_brush(value))
                elif key == 'bold':
                    result.setFontWeight(QtGui.QFont.Bold)
                elif key == 'italic':
                    result.setFontItalic(True)
                elif key == 'underline':
                    result.setUnderlineStyle(
                        QtGui.QTextCharFormat.SingleUnderline)
                elif key == 'sans':
                    result.setFontStyleHint(QtGui.QFont.SansSerif)
                elif key == 'roman':
                    result.setFontStyleHint(QtGui.QFont.Times)
                elif key == 'mono':
                    result.setFontStyleHint(QtGui.QFont.TypeWriter)
        return result

    def _get_brush(self, color):
        """ Returns a brush for the color.
        """
        result = self._brushes.get(color)
        if result is None:
            qcolor = self._get_color(color)
            result = QtGui.QBrush(qcolor)
            self._brushes[color] = result
        return result

    def _get_color(self, color):
        """ Returns a QColor built from a Pygments color string.
        """
        qcolor = QtGui.QColor()
        qcolor.setRgb(int(color[:2], base=16),
                      int(color[2:4], base=16),
                      int(color[4:6], base=16))
        return qcolor
Exemplo n.º 26
0
"""
import re
import sys
from pygments.lexers import PythonLexer
from pygments.token import Punctuation, Comment

if __name__ == "__main__":
    lexer = PythonLexer()
    pattern = re.compile(u".*(given|and_|when|then)\((.+)")
    steps = []
    print "# -*- coding: UTF-8 -*-"
    print "from pyfeature import step"
    for l in sys.stdin.xreadlines():
        m = pattern.match(l)
        if m:
            s = lexer.get_tokens(m.group(2))
            desc = ""
            for t in s:
                if t[0] != Punctuation and t[1] != u",":
                    desc += "".join(chr(c) for c in [ord(b) for b in t[1]])
                else:
                    break
            steps.append([desc.strip()])
            omit_next_token = False
            for t in s:
                if t[1] == "=":
                    omit_next_token = True
                elif not omit_next_token and t[1] != "," and t[1] != ")" and t[0] != Comment:
                    steps[-1].append(t[1])
                else:
                    omit_next_token = False
Exemplo n.º 27
0
class Console(QTextEdit):

    running = pyqtSignal()
    running_done = pyqtSignal()

    @property
    def doc(self):
        return self.document()

    @property
    def cursor(self):
        return self.textCursor()

    @property
    def root_frame(self):
        return self.doc.rootFrame()

    def unhandled_exception(self, type, value, tb):
        if type == KeyboardInterrupt:
            return
        try:
            sio = StringIO.StringIO()
            traceback.print_exception(type, value, tb, file=sio)
            fe = sio.getvalue()
            prints(fe)
            try:
                val = unicode(value)
            except:
                val = repr(value)
            msg = '<b>%s</b>:'%type.__name__ + val
            error_dialog(self, _('ERROR: Unhandled exception'), msg,
                    det_msg=fe, show=True)
        except BaseException:
            pass

    def __init__(self,
            prompt='>>> ',
            continuation='... ',
            parent=None):
        QTextEdit.__init__(self, parent)
        self.shutting_down = False
        self.compiler = CommandCompiler()
        self.buf = self.old_buf = []
        self.history = History([''], dynamic.get('console_history', []))
        self.prompt_frame = None
        self.allow_output = False
        self.prompt_frame_format = QTextFrameFormat()
        self.prompt_frame_format.setBorder(1)
        self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
        self.prompt_len = len(prompt)

        self.doc.setMaximumBlockCount(int(prefs['scrollback']))
        self.lexer = PythonLexer(ensurenl=False)
        self.tb_lexer = PythonTracebackLexer()

        self.context_menu = cm = QMenu(self) # {{{
        cm.theme = ThemeMenu(cm)
        # }}}

        self.formatter = Formatter(prompt, continuation, style=prefs['theme'])
        p = QPalette()
        p.setColor(p.Base, QColor(self.formatter.background_color))
        p.setColor(p.Text, QColor(self.formatter.color))
        self.setPalette(p)

        self.key_dispatcher = { # {{{
                Qt.Key_Enter : self.enter_pressed,
                Qt.Key_Return : self.enter_pressed,
                Qt.Key_Up : self.up_pressed,
                Qt.Key_Down : self.down_pressed,
                Qt.Key_Home : self.home_pressed,
                Qt.Key_End : self.end_pressed,
                Qt.Key_Left : self.left_pressed,
                Qt.Key_Right : self.right_pressed,
                Qt.Key_Backspace : self.backspace_pressed,
                Qt.Key_Delete : self.delete_pressed,
        } # }}}

        motd = textwrap.dedent('''\
        # Python {0}
        # {1} {2}
        '''.format(sys.version.splitlines()[0], __appname__,
            __version__))

        sys.excepthook = self.unhandled_exception

        self.controllers = []
        QTimer.singleShot(0, self.launch_controller)


        with EditBlock(self.cursor):
            self.render_block(motd)

    def shutdown(self):
        dynamic.set('console_history', self.history.serialize())
        self.shutting_down = True
        for c in self.controllers:
            c.kill()

    def contextMenuEvent(self, event):
        self.context_menu.popup(event.globalPos())
        event.accept()

    # Controller management {{{
    @property
    def controller(self):
        return self.controllers[-1]

    def no_controller_error(self):
        error_dialog(self, _('No interpreter'),
                _('No active interpreter found. Try restarting the'
                    ' console'), show=True)

    def launch_controller(self, *args):
        c = Controller(self)
        c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
        c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
        c.interpreter_died.connect(self.interpreter_died,
                type=Qt.QueuedConnection)
        c.interpreter_done.connect(self.execution_done)
        self.controllers.append(c)

    def interpreter_died(self, controller, returncode):
        if not self.shutting_down and controller.current_command is not None:
            error_dialog(self, _('Interpreter died'),
                    _('Interpreter dies while executing a command. To see '
                        'the command, click Show details'),
                    det_msg=controller.current_command, show=True)

    def execute(self, prompt_lines):
        c = self.root_frame.lastCursorPosition()
        self.setTextCursor(c)
        self.old_prompt_frame = self.prompt_frame
        self.prompt_frame = None
        self.old_buf = self.buf
        self.buf = []
        self.running.emit()
        self.controller.runsource('\n'.join(prompt_lines))

    def execution_done(self, controller, ret):
        if controller is self.controller:
            self.running_done.emit()
            if ret: # Incomplete command
                self.buf = self.old_buf
                self.prompt_frame = self.old_prompt_frame
                c = self.prompt_frame.lastCursorPosition()
                c.insertBlock()
                self.setTextCursor(c)
            else: # Command completed
                try:
                    self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
                except RuntimeError:
                    # Happens if enough lines of output that the old
                    # frame was deleted
                    pass

            self.render_current_prompt()

    # }}}

    # Prompt management {{{

    @dynamic_property
    def cursor_pos(self):
        doc = '''
        The cursor position in the prompt has the form (row, col).
        row starts at 0 for the first line
        col is 0 if the cursor is at the start of the line, 1 if it is after
        the first character, n if it is after the nth char.
        '''

        def fget(self):
            if self.prompt_frame is not None:
                pos = self.cursor.position()
                it = self.prompt_frame.begin()
                lineno = 0
                while not it.atEnd():
                    bl = it.currentBlock()
                    if bl.contains(pos):
                        return (lineno, pos - bl.position())
                    it += 1
                    lineno += 1
            return (-1, -1)

        def fset(self, val):
            row, col = val
            if self.prompt_frame is not None:
                it = self.prompt_frame.begin()
                lineno = 0
                while not it.atEnd():
                    if lineno == row:
                        c = self.cursor
                        c.setPosition(it.currentBlock().position())
                        c.movePosition(c.NextCharacter, n=col)
                        self.setTextCursor(c)
                        break
                    it += 1
                    lineno += 1

        return property(fget=fget, fset=fset, doc=doc)

    def move_cursor_to_prompt(self):
        if self.prompt_frame is not None and self.cursor_pos[0] < 0:
            c = self.prompt_frame.lastCursorPosition()
            self.setTextCursor(c)

    def prompt(self, strip_prompt_strings=True):
        if not self.prompt_frame:
            yield u'' if strip_prompt_strings else self.formatter.prompt
        else:
            it = self.prompt_frame.begin()
            while not it.atEnd():
                bl = it.currentBlock()
                t = unicode(bl.text())
                if strip_prompt_strings:
                    t = t[self.prompt_len:]
                yield t
                it += 1

    def set_prompt(self, lines):
        self.render_current_prompt(lines)

    def clear_current_prompt(self):
        if self.prompt_frame is None:
            c = self.root_frame.lastCursorPosition()
            self.prompt_frame = c.insertFrame(self.prompt_frame_format)
            self.setTextCursor(c)
        else:
            c = self.prompt_frame.firstCursorPosition()
            self.setTextCursor(c)
            c.setPosition(self.prompt_frame.lastPosition(), c.KeepAnchor)
            c.removeSelectedText()
            c.setPosition(self.prompt_frame.firstPosition())

    def render_current_prompt(self, lines=None, restore_cursor=False):
        row, col = self.cursor_pos
        cp = list(self.prompt()) if lines is None else lines
        self.clear_current_prompt()

        for i, line in enumerate(cp):
            start = i == 0
            end = i == len(cp) - 1
            self.formatter.render_prompt(not start, self.cursor)
            self.formatter.render(self.lexer.get_tokens(line), self.cursor)
            if not end:
                self.cursor.insertBlock()

        if row > -1 and restore_cursor:
            self.cursor_pos = (row, col)

        self.ensureCursorVisible()

    # }}}

    # Non-prompt Rendering {{{

    def render_block(self, text, restore_prompt=True):
        self.formatter.render(self.lexer.get_tokens(text), self.cursor)
        self.cursor.insertBlock()
        self.cursor.movePosition(self.cursor.End)
        if restore_prompt:
            self.render_current_prompt()

    def show_error(self, is_syntax_err, tb, controller=None):
        if self.prompt_frame is not None:
            # At a prompt, so redirect output
            return prints(tb, end='')
        try:
            self.buf.append(tb)
            if is_syntax_err:
                self.formatter.render_syntax_error(tb, self.cursor)
            else:
                self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
        except:
            prints(tb, end='')
        self.ensureCursorVisible()
        QApplication.processEvents()

    def show_output(self, raw, which='stdout', controller=None):
        def do_show():
            try:
                self.buf.append(raw)
                self.formatter.render_raw(raw, self.cursor)
            except:
                import traceback
                prints(traceback.format_exc())
                prints(raw, end='')

        if self.prompt_frame is not None:
            with Prepender(self):
                do_show()
        else:
            do_show()
        self.ensureCursorVisible()
        QApplication.processEvents()

    # }}}

    # Keyboard management {{{

    def keyPressEvent(self, ev):
        text = unicode(ev.text())
        key = ev.key()
        action = self.key_dispatcher.get(key, None)
        if callable(action):
            action()
        elif key in (Qt.Key_Escape,):
            QTextEdit.keyPressEvent(self, ev)
        elif text:
            self.text_typed(text)
        else:
            QTextEdit.keyPressEvent(self, ev)

    def left_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if pos > self.prompt_len:
            c = self.cursor
            c.movePosition(c.PreviousCharacter)
            self.setTextCursor(c)
        elif lineno > 0:
            c = self.cursor
            c.movePosition(c.Up)
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
        self.ensureCursorVisible()

    def up_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if lineno == 0:
            b = self.history.back()
            if b is not None:
                self.set_prompt(b)
        else:
            c = self.cursor
            c.movePosition(c.Up)
            self.setTextCursor(c)
        self.ensureCursorVisible()


    def backspace_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if pos > self.prompt_len:
            self.cursor.deletePreviousChar()
        elif lineno > 0:
            c = self.cursor
            c.movePosition(c.Up)
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
        self.ensureCursorVisible()

    def delete_pressed(self):
        self.cursor.deleteChar()
        self.ensureCursorVisible()

    def right_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        c = self.cursor
        cp = list(self.prompt(False))
        if pos < len(cp[lineno]):
            c.movePosition(c.NextCharacter)
        elif lineno < len(cp)-1:
            c.movePosition(c.NextCharacter, n=1+self.prompt_len)
        self.setTextCursor(c)
        self.ensureCursorVisible()

    def down_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        c = self.cursor
        cp = list(self.prompt(False))
        if lineno >= len(cp) - 1:
            b = self.history.forward()
            if b is not None:
                self.set_prompt(b)
        else:
            c = self.cursor
            c.movePosition(c.Down)
            self.setTextCursor(c)
        self.ensureCursorVisible()


    def home_pressed(self):
        if self.prompt_frame is not None:
            mods = QApplication.keyboardModifiers()
            ctrl = bool(int(mods & Qt.CTRL))
            if ctrl:
                self.cursor_pos = (0, self.prompt_len)
            else:
                c = self.cursor
                c.movePosition(c.StartOfLine)
                c.movePosition(c.NextCharacter, n=self.prompt_len)
                self.setTextCursor(c)
            self.ensureCursorVisible()

    def end_pressed(self):
        if self.prompt_frame is not None:
            mods = QApplication.keyboardModifiers()
            ctrl = bool(int(mods & Qt.CTRL))
            if ctrl:
                self.cursor_pos = (len(list(self.prompt()))-1, self.prompt_len)
            c = self.cursor
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
            self.ensureCursorVisible()

    def enter_pressed(self):
        if self.prompt_frame is None:
            return
        if not self.controller.is_alive:
            return self.no_controller_error()
        cp = list(self.prompt())
        if cp[0]:
            try:
                ret = self.compiler('\n'.join(cp))
            except:
                pass
            else:
                if ret is None:
                    c = self.prompt_frame.lastCursorPosition()
                    c.insertBlock()
                    self.setTextCursor(c)
                    self.render_current_prompt()
                    return
                else:
                    self.history.enter(cp)
            self.execute(cp)

    def text_typed(self, text):
        if self.prompt_frame is not None:
            self.move_cursor_to_prompt()
            self.cursor.insertText(text)
            self.render_current_prompt(restore_cursor=True)
            self.history.current = list(self.prompt())
Exemplo n.º 28
0
class Console(QTextEdit):

    running = pyqtSignal()
    running_done = pyqtSignal()

    @property
    def doc(self):
        return self.document()

    @property
    def cursor(self):
        return self.textCursor()

    @property
    def root_frame(self):
        return self.doc.rootFrame()

    def unhandled_exception(self, type, value, tb):
        if type == KeyboardInterrupt:
            return
        try:
            sio = StringIO.StringIO()
            traceback.print_exception(type, value, tb, file=sio)
            fe = sio.getvalue()
            prints(fe)
            try:
                val = unicode(value)
            except:
                val = repr(value)
            msg = '<b>%s</b>:'%type.__name__ + val
            error_dialog(self, _('ERROR: Unhandled exception'), msg,
                    det_msg=fe, show=True)
        except BaseException:
            pass

    def __init__(self,
            prompt='>>> ',
            continuation='... ',
            parent=None):
        QTextEdit.__init__(self, parent)
        self.shutting_down = False
        self.compiler = CommandCompiler()
        self.buf = self.old_buf = []
        self.history = History([''], dynamic.get('console_history', []))
        self.prompt_frame = None
        self.allow_output = False
        self.prompt_frame_format = QTextFrameFormat()
        self.prompt_frame_format.setBorder(1)
        self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
        self.prompt_len = len(prompt)

        self.doc.setMaximumBlockCount(int(prefs['scrollback']))
        self.lexer = PythonLexer(ensurenl=False)
        self.tb_lexer = PythonTracebackLexer()

        self.context_menu = cm = QMenu(self) # {{{
        cm.theme = ThemeMenu(cm)
        # }}}

        self.formatter = Formatter(prompt, continuation, style=prefs['theme'])
        p = QPalette()
        p.setColor(p.Base, QColor(self.formatter.background_color))
        p.setColor(p.Text, QColor(self.formatter.color))
        self.setPalette(p)

        self.key_dispatcher = { # {{{
                Qt.Key_Enter : self.enter_pressed,
                Qt.Key_Return : self.enter_pressed,
                Qt.Key_Up : self.up_pressed,
                Qt.Key_Down : self.down_pressed,
                Qt.Key_Home : self.home_pressed,
                Qt.Key_End : self.end_pressed,
                Qt.Key_Left : self.left_pressed,
                Qt.Key_Right : self.right_pressed,
                Qt.Key_Backspace : self.backspace_pressed,
                Qt.Key_Delete : self.delete_pressed,
        } # }}}

        motd = textwrap.dedent('''\
        # Python {0}
        # {1} {2}
        '''.format(sys.version.splitlines()[0], __appname__,
            __version__))

        sys.excepthook = self.unhandled_exception

        self.controllers = []
        QTimer.singleShot(0, self.launch_controller)


        with EditBlock(self.cursor):
            self.render_block(motd)

    def shutdown(self):
        dynamic.set('console_history', self.history.serialize())
        self.shutting_down = True
        for c in self.controllers:
            c.kill()

    def contextMenuEvent(self, event):
        self.context_menu.popup(event.globalPos())
        event.accept()

    # Controller management {{{
    @property
    def controller(self):
        return self.controllers[-1]

    def no_controller_error(self):
        error_dialog(self, _('No interpreter'),
                _('No active interpreter found. Try restarting the'
                    ' console'), show=True)

    def launch_controller(self, *args):
        c = Controller(self)
        c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
        c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
        c.interpreter_died.connect(self.interpreter_died,
                type=Qt.QueuedConnection)
        c.interpreter_done.connect(self.execution_done)
        self.controllers.append(c)

    def interpreter_died(self, controller, returncode):
        if not self.shutting_down and controller.current_command is not None:
            error_dialog(self, _('Interpreter died'),
                    _('Interpreter dies while executing a command. To see '
                        'the command, click Show details'),
                    det_msg=controller.current_command, show=True)

    def execute(self, prompt_lines):
        c = self.root_frame.lastCursorPosition()
        self.setTextCursor(c)
        self.old_prompt_frame = self.prompt_frame
        self.prompt_frame = None
        self.old_buf = self.buf
        self.buf = []
        self.running.emit()
        self.controller.runsource('\n'.join(prompt_lines))

    def execution_done(self, controller, ret):
        if controller is self.controller:
            self.running_done.emit()
            if ret: # Incomplete command
                self.buf = self.old_buf
                self.prompt_frame = self.old_prompt_frame
                c = self.prompt_frame.lastCursorPosition()
                c.insertBlock()
                self.setTextCursor(c)
            else: # Command completed
                try:
                    self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
                except RuntimeError:
                    # Happens if enough lines of output that the old
                    # frame was deleted
                    pass

            self.render_current_prompt()

    # }}}

    # Prompt management {{{

    @dynamic_property
    def cursor_pos(self):
        doc = '''
        The cursor position in the prompt has the form (row, col).
        row starts at 0 for the first line
        col is 0 if the cursor is at the start of the line, 1 if it is after
        the first character, n if it is after the nth char.
        '''

        def fget(self):
            if self.prompt_frame is not None:
                pos = self.cursor.position()
                it = self.prompt_frame.begin()
                lineno = 0
                while not it.atEnd():
                    bl = it.currentBlock()
                    if bl.contains(pos):
                        return (lineno, pos - bl.position())
                    it += 1
                    lineno += 1
            return (-1, -1)

        def fset(self, val):
            row, col = val
            if self.prompt_frame is not None:
                it = self.prompt_frame.begin()
                lineno = 0
                while not it.atEnd():
                    if lineno == row:
                        c = self.cursor
                        c.setPosition(it.currentBlock().position())
                        c.movePosition(c.NextCharacter, n=col)
                        self.setTextCursor(c)
                        break
                    it += 1
                    lineno += 1

        return property(fget=fget, fset=fset, doc=doc)

    def move_cursor_to_prompt(self):
        if self.prompt_frame is not None and self.cursor_pos[0] < 0:
            c = self.prompt_frame.lastCursorPosition()
            self.setTextCursor(c)

    def prompt(self, strip_prompt_strings=True):
        if not self.prompt_frame:
            yield u'' if strip_prompt_strings else self.formatter.prompt
        else:
            it = self.prompt_frame.begin()
            while not it.atEnd():
                bl = it.currentBlock()
                t = unicode(bl.text())
                if strip_prompt_strings:
                    t = t[self.prompt_len:]
                yield t
                it += 1

    def set_prompt(self, lines):
        self.render_current_prompt(lines)

    def clear_current_prompt(self):
        if self.prompt_frame is None:
            c = self.root_frame.lastCursorPosition()
            self.prompt_frame = c.insertFrame(self.prompt_frame_format)
            self.setTextCursor(c)
        else:
            c = self.prompt_frame.firstCursorPosition()
            self.setTextCursor(c)
            c.setPosition(self.prompt_frame.lastPosition(), c.KeepAnchor)
            c.removeSelectedText()
            c.setPosition(self.prompt_frame.firstPosition())

    def render_current_prompt(self, lines=None, restore_cursor=False):
        row, col = self.cursor_pos
        cp = list(self.prompt()) if lines is None else lines
        self.clear_current_prompt()

        for i, line in enumerate(cp):
            start = i == 0
            end = i == len(cp) - 1
            self.formatter.render_prompt(not start, self.cursor)
            self.formatter.render(self.lexer.get_tokens(line), self.cursor)
            if not end:
                self.cursor.insertBlock()

        if row > -1 and restore_cursor:
            self.cursor_pos = (row, col)

        self.ensureCursorVisible()

    # }}}

    # Non-prompt Rendering {{{

    def render_block(self, text, restore_prompt=True):
        self.formatter.render(self.lexer.get_tokens(text), self.cursor)
        self.cursor.insertBlock()
        self.cursor.movePosition(self.cursor.End)
        if restore_prompt:
            self.render_current_prompt()

    def show_error(self, is_syntax_err, tb, controller=None):
        if self.prompt_frame is not None:
            # At a prompt, so redirect output
            return prints(tb, end='')
        try:
            self.buf.append(tb)
            if is_syntax_err:
                self.formatter.render_syntax_error(tb, self.cursor)
            else:
                self.formatter.render(self.tb_lexer.get_tokens(tb), self.cursor)
        except:
            prints(tb, end='')
        self.ensureCursorVisible()
        QApplication.processEvents()

    def show_output(self, raw, which='stdout', controller=None):
        def do_show():
            try:
                self.buf.append(raw)
                self.formatter.render_raw(raw, self.cursor)
            except:
                import traceback
                prints(traceback.format_exc())
                prints(raw, end='')

        if self.prompt_frame is not None:
            with Prepender(self):
                do_show()
        else:
            do_show()
        self.ensureCursorVisible()
        QApplication.processEvents()

    # }}}

    # Keyboard management {{{

    def keyPressEvent(self, ev):
        text = unicode(ev.text())
        key = ev.key()
        action = self.key_dispatcher.get(key, None)
        if callable(action):
            action()
        elif key in (Qt.Key_Escape,):
            QTextEdit.keyPressEvent(self, ev)
        elif text:
            self.text_typed(text)
        else:
            QTextEdit.keyPressEvent(self, ev)

    def left_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if pos > self.prompt_len:
            c = self.cursor
            c.movePosition(c.PreviousCharacter)
            self.setTextCursor(c)
        elif lineno > 0:
            c = self.cursor
            c.movePosition(c.Up)
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
        self.ensureCursorVisible()

    def up_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if lineno == 0:
            b = self.history.back()
            if b is not None:
                self.set_prompt(b)
        else:
            c = self.cursor
            c.movePosition(c.Up)
            self.setTextCursor(c)
        self.ensureCursorVisible()


    def backspace_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        if pos > self.prompt_len:
            self.cursor.deletePreviousChar()
        elif lineno > 0:
            c = self.cursor
            c.movePosition(c.Up)
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
        self.ensureCursorVisible()

    def delete_pressed(self):
        self.cursor.deleteChar()
        self.ensureCursorVisible()

    def right_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        c = self.cursor
        cp = list(self.prompt(False))
        if pos < len(cp[lineno]):
            c.movePosition(c.NextCharacter)
        elif lineno < len(cp)-1:
            c.movePosition(c.NextCharacter, n=1+self.prompt_len)
        self.setTextCursor(c)
        self.ensureCursorVisible()

    def down_pressed(self):
        lineno, pos = self.cursor_pos
        if lineno < 0: return
        c = self.cursor
        cp = list(self.prompt(False))
        if lineno >= len(cp) - 1:
            b = self.history.forward()
            if b is not None:
                self.set_prompt(b)
        else:
            c = self.cursor
            c.movePosition(c.Down)
            self.setTextCursor(c)
        self.ensureCursorVisible()


    def home_pressed(self):
        if self.prompt_frame is not None:
            mods = QApplication.keyboardModifiers()
            ctrl = bool(int(mods & Qt.CTRL))
            if ctrl:
                self.cursor_pos = (0, self.prompt_len)
            else:
                c = self.cursor
                c.movePosition(c.StartOfLine)
                c.movePosition(c.NextCharacter, n=self.prompt_len)
                self.setTextCursor(c)
            self.ensureCursorVisible()

    def end_pressed(self):
        if self.prompt_frame is not None:
            mods = QApplication.keyboardModifiers()
            ctrl = bool(int(mods & Qt.CTRL))
            if ctrl:
                self.cursor_pos = (len(list(self.prompt()))-1, self.prompt_len)
            c = self.cursor
            c.movePosition(c.EndOfLine)
            self.setTextCursor(c)
            self.ensureCursorVisible()

    def enter_pressed(self):
        if self.prompt_frame is None:
            return
        if not self.controller.is_alive:
            return self.no_controller_error()
        cp = list(self.prompt())
        if cp[0]:
            try:
                ret = self.compiler('\n'.join(cp))
            except:
                pass
            else:
                if ret is None:
                    c = self.prompt_frame.lastCursorPosition()
                    c.insertBlock()
                    self.setTextCursor(c)
                    self.render_current_prompt()
                    return
                else:
                    self.history.enter(cp)
            self.execute(cp)

    def text_typed(self, text):
        if self.prompt_frame is not None:
            self.move_cursor_to_prompt()
            self.cursor.insertText(text)
            self.render_current_prompt(restore_cursor=True)
            self.history.current = list(self.prompt())