예제 #1
0
    def random_insertion(self):
        """Insert random characters at random locations within a program."""
        print "Running random_insert on {}".format(self.filename)
        self.reset()

        self.treemanager.import_file(self.program)
        assert self.parser.last_status == True

        self.text_compare(self.program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.log.append("self.treemanager.cursor_reset()")
                self.log.append("self.move(DOWN, %s)" % linenr)
                self.log.append("self.move(RIGHT, %s)" % col)
                self.treemanager.cursor_reset()
                self.move(DOWN, linenr)
                self.move(RIGHT, col)
                k = self.get_random_key()
                self.log.append("self.treemanager.key_normal(%s)" % repr(k))
                x = self.treemanager.key_normal(k)
                if x == "eos":
                    continue
            self.log.append("self.treemanager.undo_snapshot()")
            self.treemanager.undo_snapshot()

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.import_file(self.program)

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)
    def find_expressions(self, program, expr):
        parser, lexer = self.sub.load()
        treemanager = TreeManager()
        treemanager.add_parser(parser, lexer, self.sub.name)
        treemanager.import_file(program)

        # find all sub expressions
        l = self.find_nonterms_by_name(treemanager, expr)
        return [subtree_to_text(st).rstrip() for st in l]
def find_subtree(grammar, program, cond):
    parser, lexer = grammar.load()
    treemanager = TreeManager()
    treemanager.add_parser(parser, lexer, grammar.name)
    try:
        treemanager.import_file(program)
    except Exception:
        return None
    if parser.last_status is False:
        return None

    # find all sub expressions
    l = find_nonterms_by_name(treemanager, cond)
    return [subtree_to_text(st).rstrip() for st in l]
예제 #4
0
def run(filename):
    # Load grammar
    parser, lexer = java.load()
    treemanager = TreeManager()
    treemanager.add_parser(parser, lexer, "")

    # Load file
    print("Running", filename)
    with open(filename, "r") as f:
        treemanager.import_file(f.read())

    if not parser.last_status:
        print("Couldn't parse file. Skip.")
        return

    filesize = os.path.getsize(filename)

    timings = []

    # Insert `1+` after every `=`
    node = treemanager.get_bos()
    while type(node) is not EOS:
        if node.symbol.name == "=":
            node.insert("1+", len(node.symbol.name))
            node.mark_changed()
            treemanager.relex(node)
            treemanager.post_keypress("")
            treemanager.save_current_version()
            parser.prev_version = treemanager.version
            parser.reference_version = treemanager.reference_version
            start = time()
            parser.inc_parse()
            end = time()
            parser.top_down_reuse()
            treemanager.save_current_version(
                postparse=True)  # save post parse tree
            if parser.last_status:
                treemanager.reference_version = treemanager.version
            TreeManager.version = treemanager.version

            if not parser.last_status:
                break
            timings.append(str(end - start))
        node = node.next_term

    if len(timings) > 0:
        with open("results.csv", "a+") as f:
            f.write("{} {} {}".format(filename, filesize, ",".join(timings)))
            f.write("\n")
예제 #5
0
class BootstrapParser(object):
    def __init__(self, lr_type=1, whitespaces=False):
        self.lr_type = lr_type
        self.whitespaces = whitespaces
        # load (old) parser for grammar grammar
        self.rules = {}
        self.lrules = []
        self.start_symbol = None
        self.incparser = None
        self.inclexer = None
        self.terminals = set()
        self.extra_alternatives = {}
        self.change_startrule = None
        self.options = {"nowhitespace": []}
        self.precedences = []
        self.current_rulename = ""
        self.all_terminals = set()
        self.functions = []
        self.prod_ids = {}

    def implicit_ws(self):
        if self.options.has_key("implicit_ws"):
            if self.options["implicit_ws"] == "true":
                return True
        return False

    def implicit_newlines(self):
        if self.options.has_key("implicit_newlines"):
            if self.options["implicit_newlines"] == "false":
                return False
        return True

    def indentation_based(self):
        if self.options.has_key("indentation"):
            if self.options["indentation"] == "true":
                return True
        return False

    def parse(self, ecogrammar):
        # this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco))
        from grammars.eco_grammar import eco_grammar as grammar
        self.lexer = IncrementalLexer(grammar.priorities)
        self.parser = IncParser(grammar.grammar, 1, True)
        self.parser.init_ast()
        self.ast = self.parser.previous_version.parent
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, grammar.name)
        self.treemanager.import_file(ecogrammar)
        if self.parser.last_status == False:
            raise Exception("Invalid input grammar due to syntax errors")
        self.read_options()
        self.parse_both()
        self.create_parser()
        self.create_lexer()

    def parse_both(self):
        # parse rules
        startrule = self.ast.children[1]  # startrule
        grammar = startrule.children[1]
        parser = grammar.children[0]
        assert parser.symbol.name == "parser"
        self.parse_rules(parser)

        # parse lexer
        startrule = self.ast.children[1]  # startrule
        grammar = startrule.children[1]
        for element in grammar.children:
            if element.symbol.name == "lexer":
                break
        lexer = element
        assert lexer.symbol.name == "lexer"
        self.parse_lexer(lexer)
        for name, regex in self.lrules:
            # collect terminals for parser modifications
            self.all_terminals.add(name)

    def read_options(self):
        startrule = self.ast.children[1]  # startrule
        assert startrule.symbol.name == "Startrule"
        grammar = startrule.children[1]
        assert grammar.symbol.name == "grammar"
        for element in grammar.children:
            if element.symbol.name == "options":
                break
        if element.symbol.name != "options":
            # grammar has no options
            print("warning: grammar has no options")
            # backwards compatibility
            if self.whitespaces:
                self.options["implicit_ws"] = "true"
            return
        options = element
        assert options.symbol.name == "options"
        self.parse_options(options)

    def parse_options(self, options):
        if options.children == []:
            return
        if len(options.children) > 0:
            assert options.children[0].symbol.name == "settings"
            self.parse_settings(options.children[0])
        if len(options.children) > 1:
            assert options.children[1].symbol.name == "precedences"
            self.parse_precedences(options.children[1])

    def parse_settings(self, options):
        if options.children == []:
            return
        if len(options.children) == 2:
            more = options.children[0]
            self.parse_settings(more)
            option = options.children[1]
        else:
            option = options.children[0]
        name = option.children[2].symbol.name
        choice = option.children[6]
        assert choice.symbol.name == "choice"
        if choice.children[0].symbol.name == "choice_list":
            self.options[name] = self.parse_choicelist(choice.children[0])
        else:
            self.options[name] = choice.children[0].symbol.name

    def parse_choicelist(self, symbol):
        s = []
        for c in symbol.children:
            if c.symbol.name == ",":
                continue
            if c.symbol.name == "WS":
                continue
            if c.lookup == "nonterminal":
                s.append(c.symbol.name)
                continue
            if c.symbol.name == "choice_list":
                rec_s = self.parse_choicelist(symbol.children[0])
                s.extend(rec_s)
                continue
        return s

    def parse_precedences(self, precedences):
        if precedences.children == []:
            return
        # recursively parse other precedences
        if len(precedences.children) == 2:
            more = precedences.children[0]
            self.parse_precedences(more)
            precedence = precedences.children[1]
        else:
            precedence = precedences.children[0]
        # parse single precedence
        name = precedence.children[0].symbol.name
        terminals = self.parse_precedence_symbols(precedence.children[2])
        self.precedences.append((name, terminals))

    def parse_precedence_symbols(self, symbol):
        s = []
        for c in symbol.children:
            if c.symbol.name == "WS":
                continue
            if c.symbol.name == "terminals":
                rec_s = self.parse_precedence_symbols(symbol.children[0])
                s.extend(rec_s)
            if c.lookup == "terminal":
                s.append(c.symbol.name[1:-1])
        return s

    def create_parser(self, pickle_id=None):
        self.all_terminals.update(self.terminals)

        for fname, terminals, parentrule in self.functions:
            if fname.startswith("*match_until"):
                if Nonterminal(fname) not in self.rules:
                    r = Rule(Nonterminal(fname))
                    for t in self.all_terminals:
                        if t not in terminals:
                            r.add_alternative(
                                [Nonterminal(fname),
                                 Terminal(t)], None, t)
                    r.add_alternative([])
                    self.rules[r.symbol] = r
                # remove whitespace before special rule from parent rule, e.g.
                # multistring ::= "MLS" WS *match_until "MLS" WS
                #                       ^ this WS causes shift/reduce conflicts
                prule = self.rules[Nonterminal(parentrule)]
                for a in prule.alternatives:
                    for i in range(len(a)):
                        sym = a[i]
                        if sym.name == "WS":
                            if len(a) > i + 1 and a[i + 1].name.startswith(
                                    "*match_until"):
                                a.pop(i)
                                break

        if self.implicit_ws():
            ws_rule = Rule()
            ws_rule.symbol = Nonterminal("WS")
            ws_rule.add_alternative([Nonterminal("WS"), Terminal("<ws>")])
            # get comment rule
            if self.options.has_key('comment_rule'):
                cmt_rules = self.options['comment_rule']
                for cmt_rule in cmt_rules:
                    if Nonterminal(cmt_rule) in self.rules:
                        ws_rule.add_alternative(
                            [Nonterminal("WS"),
                             Nonterminal("comment")])
            if self.implicit_newlines():
                ws_rule.add_alternative(
                    [Nonterminal("WS"),
                     Terminal("<return>")])
                ws_rule.add_alternative([
                    Nonterminal("WS"),
                    Terminal("<backslash>"),
                    Terminal("<return>")
                ])
            ws_rule.add_alternative([])  # or empty
            self.rules[ws_rule.symbol] = ws_rule
            for a in ws_rule.alternatives:
                self.prod_ids[Production(ws_rule.symbol,
                                         a)] = len(self.prod_ids)

            # allow whitespace/comments at beginning of file
            start_rule = Rule()
            start_rule.symbol = Nonterminal("Startrule")
            start_rule.add_alternative([Nonterminal("WS"), self.start_symbol])
            self.rules[start_rule.symbol] = start_rule
            self.prod_ids[Production(start_rule.symbol,
                                     start_rule.alternatives[0])] = len(
                                         self.prod_ids)
            self.start_symbol = start_rule.symbol

        incparser = IncParser()
        incparser.from_dict(self.rules, self.start_symbol, self.lr_type,
                            self.implicit_ws(), pickle_id, self.precedences,
                            self.prod_ids)
        incparser.init_ast()
        self.incparser = incparser

    def parse_rules(self, node):
        if node.children[0].symbol.name == "parser":
            self.parse_rules(node.children[0])
            self.parse_rule(node.children[3])
        elif node.children[0].symbol.name == "rule":
            self.parse_rule(node.children[0])

    def parse_rule(self, node):
        name = node.children[0].symbol.name
        self.current_rulename = name
        alternatives = self.parse_alternatives(node.children[4])
        symbol = Nonterminal(name)
        if self.start_symbol is None:
            self.start_symbol = symbol
        if self.change_startrule and symbol.name == self.change_startrule:
            self.start_symbol = symbol
        r = Rule(symbol)
        for a in alternatives:
            r.add_alternative(a[0], a[1], a[2])
            self.prod_ids[Production(symbol, a[0])] = len(self.prod_ids)
        # add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes)
        if self.extra_alternatives.has_key(symbol.name):
            for n in self.extra_alternatives[symbol.name]:
                a = [MagicTerminal(n), Nonterminal("WS")]
                r.add_alternative(a)
                self.prod_ids[Production(symbol, a)] = len(self.prod_ids)
        self.rules[symbol] = r

    def parse_alternatives(self, node):
        if node.children[0].symbol.name == "alternatives":
            alternatives = self.parse_alternatives(node.children[0])
            alternative = self.parse_alternative(node.children[3])
            alternatives.append(alternative)
            return alternatives
        elif node.children[0].symbol.name == "right":
            return [self.parse_alternative(node.children[0])]

    def parse_alternative(self, node):
        if len(node.children) > 0:
            annotation = None
            prec = None
            for c in node.children:
                if c.symbol.name == "symbols":
                    symbols = self.parse_symbols(c)
                if c.symbol.name == "prec":
                    prec = self.parse_prec(c)
                if c.symbol.name == "annotations":
                    annotation = self.parse_annotation(c)
            return (symbols, annotation, prec)
        else:
            return ([], None, None)

    def parse_prec(self, node):
        if node.children:
            c = node.children[2]
            return c.symbol.name[1:-1]

    def parse_symbols(self, node):
        if node.children[0].symbol.name == "symbols":
            symbols = self.parse_symbols(node.children[0])
            symbol = self.parse_symbol(node.children[1])
            symbols.append(symbol)
            if (
                    isinstance(symbol, Terminal)
                    or isinstance(symbol, MagicTerminal)
            ) and self.implicit_ws(
            ) and self.current_rulename not in self.options["nowhitespace"]:
                symbols.append(Nonterminal("WS"))
            return symbols
        elif node.children[0].symbol.name == "symbol":
            l = []
            symbol = self.parse_symbol(node.children[0])
            l.append(symbol)
            if isinstance(symbol, Terminal) and self.implicit_ws(
            ) and self.current_rulename not in self.options["nowhitespace"]:
                l.append(Nonterminal("WS"))
            return l

    def parse_symbol(self, node):
        node = node.children[0]
        if node.lookup == "nonterminal":
            return Nonterminal(node.symbol.name)
        elif node.lookup == "terminal":
            if node.symbol.name != "\"<eos>\"":
                self.terminals.add(node.symbol.name[1:-1])
            return Terminal(node.symbol.name[1:-1])
        elif node.lookup == "languagebox":
            return MagicTerminal(node.symbol.name)
        elif node.symbol.name == "function":
            return self.parse_function(node)

    def parse_function(self, node):
        fname = node.children[0].symbol.name
        terminals = self.parse_fargs(node.children[4])
        safe_name = "*%s%s" % (fname, hash(frozenset(terminals)))
        self.functions.append((safe_name, terminals, self.current_rulename))
        return Nonterminal(safe_name)

    def parse_fargs(self, symbol):
        s = []
        for c in symbol.children:
            if c.symbol.name == ",":
                continue
            if c.symbol.name == "WS":
                continue
            if c.lookup == "terminal":
                s.append(c.symbol.name[1:-1])
                continue
            if c.symbol.name == "f_args":
                rec_s = self.parse_fargs(symbol.children[0])
                s.extend(rec_s)
        return s

    def parse_annotation(self, node):
        a_options = node.children[2]
        assert a_options.symbol.name == "a_options"
        if a_options.children[0].symbol.name == "astnode":
            return self.parse_astnode(a_options.children[0])
        elif a_options.children[0].symbol.name == "expression":
            return self.parse_expression(a_options.children[0])
        elif a_options.children[0].symbol.name == "forloop":
            return self.parse_foreach(a_options.children[0])

    def parse_astnode(self, node):
        name = node.children[0].symbol.name
        children = self.parse_astnode_children(node.children[4])
        d = {}
        for n, expr in children:
            d[n] = expr
        return AstNode(name, d)

    def parse_astnode_children(self, node):
        assert node.symbol.name == "astnode_children"
        if node.children[0].symbol.name == "astnode_child":
            return [self.parse_astnode_child(node.children[0])]
        elif node.children[0].symbol.name == "astnode_children":
            children = self.parse_astnode_children(node.children[0])
            child = self.parse_astnode_child(node.children[3])
            children.append(child)
            return children

    def parse_astnode_child(self, node):
        assert node.symbol.name == "astnode_child"
        name = node.children[0].symbol.name
        if node.children[4].symbol.name == "expression":
            expr = self.parse_expression(node.children[4])
        elif node.children[4].symbol.name == "reference":
            expr = self.parse_reference(node.children[4])
        return (name, expr)

    def parse_expression(self, node):
        if node.children[0].symbol.name == "node":
            return self.parse_node(node.children[0])
        elif node.children[0].symbol.name == "list":
            return self.parse_list(node.children[0])
        elif node.children[0].symbol.name == "node_ref":
            return self.parse_noderef(node.children[0])
        else:
            expr1 = self.parse_expression(node.children[0])
            if node.children[3].symbol.name == "node":
                expr2 = self.parse_node(node.children[3])
            else:
                expr2 = self.parse_list(node.children[3])
            return AddExpr(expr1, expr2)

    def parse_foreach(self, node):
        item = self.parse_node(node.children[4])
        expr = self.parse_astnode(node.children[7])
        return Foreach(node.symbol.name, item, expr)

    def parse_noderef(self, node):
        lookup = self.parse_node(node.children[0])
        attr = node.children[3]
        lookup.attribute = attr.symbol.name
        return lookup

    def parse_node(self, node):
        return LookupExpr(int(node.children[2].symbol.name))

    def parse_list(self, node):
        return ListExpr(self.parse_listloop(node.children[2]))

    def parse_reference(self, node):
        base = node.children[0].symbol.name
        ref = node.children[4].symbol.name
        return ReferenceExpr(base, ref)

    def parse_listloop(self, node):
        if len(node.children) == 0:
            return []
        if node.children[0].symbol.name == "list_loop":
            l = self.parse_listloop(node.children[0])
            element = self.parse_unknown(node.children[3])
            l.append(element)
            return l
        else:
            return [self.parse_unknown(node.children[0])]

    def parse_unknown(self, node):
        if node.symbol.name == "node":
            return self.parse_node(node)
        elif node.symbol.name == "astnode":
            return self.parse_astnode(node)

    def create_lexer(self, buildlexer=True):
        names = []
        regexs = []
        for name, regex in self.lrules:
            names.append(name)
            self.all_terminals.add(name)
            regexs.append(regex)
        # add so far undefined terminals
        undefined_terminals = self.terminals.difference(set(names))
        import re
        for t in undefined_terminals:
            names.insert(0, t)
            regexs.insert(0, re.escape(t))
        if not buildlexer:
            self.inclexer = (names, regexs)
            return
        self.inclexer = IncrementalLexerCF()
        self.inclexer.from_name_and_regex(names, regexs)
        if self.indentation_based():
            self.inclexer.indentation_based = True

    def parse_lexer(self, lexer):
        if lexer.children[0].symbol.name == "lrule":
            self.parse_lrule(lexer.children[0])
        elif lexer.children[0].symbol.name == "lexer":
            self.parse_lexer(lexer.children[0])
            self.parse_lrule(lexer.children[1])

    def parse_lrule(self, lrule):
        assert lrule.children[0].symbol.name == "tokenname"
        name = lrule.children[0].children[0].symbol.name
        regex = lrule.children[3].symbol.name[1:-1]
        self.lrules.append((name, regex))
예제 #6
0
class Test_MultiTextNodePython:

    def setup_class(cls):
        parser, lexer = python.load()
        cls.lexer = lexer
        cls.parser = parser
        cls.parser.init_ast()
        cls.ast = cls.parser.previous_version
        cls.treemanager = TreeManager()
        cls.treemanager.add_parser(cls.parser, cls.lexer, python.name)

        cls.treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite

    def reset(self):
        self.parser.reset()
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, calc.name)
        self.treemanager.set_font_test(7, 17)

    def test_simple(self):
        self.reset()
        inputstring = "x = \"\"\"abcdef\"\"\""
        for c in inputstring:
            self.treemanager.key_normal(c)

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

    def test_relex_over_indentation(self):
        self.reset()
        inputstring = """class X:
    x = 1
    def x():
        pass
    y = 2"""

        self.treemanager.import_file(inputstring)

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(DOWN)
        self.treemanager.cursor_movement(DOWN)
        self.treemanager.cursor_movement(DOWN)
        self.treemanager.key_end()

        assert self.treemanager.cursor.node.symbol.name == "pass"

        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")

        self.treemanager.cursor_movement(UP)
        self.treemanager.cursor_movement(UP)
        self.treemanager.key_end()

        assert self.treemanager.cursor.node.symbol.name == "1"
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")

        assert self.parser.last_status == True

    def test_indentation_to_string_and_back(self):
        self.reset()
        inputstring = """class X:
    a
    b"""

        self.treemanager.import_file(inputstring)

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(DOWN)
        self.treemanager.cursor_movement(DOWN)
        self.treemanager.key_end()

        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")

        self.treemanager.cursor_movement(UP)
        self.treemanager.cursor_movement(UP)
        self.treemanager.key_home()

        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("\"")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(DOWN)
        self.treemanager.cursor_movement(DOWN)
        self.treemanager.key_end()
        self.treemanager.key_backspace()

        assert self.parser.last_status == False

    def test_remember_open_lexing_states(self):

        self.reset()
        inputstring = """x = 1
y = 2"""

        self.treemanager.import_file(inputstring)

        assert self.parser.last_status == True

        self.treemanager.key_end()
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.key_normal("\"")
        #assert self.parser.last_status == False # unfinished lexing jobs

        self.treemanager.key_end()
        self.treemanager.key_normal("\"")
        assert self.parser.last_status == True

    def test_triplequote_string(self):

        self.reset()
        inputstring = 'x="""abc"""'

        for i in inputstring:
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""abc"""'

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.key_normal("\"")

        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '""'
        assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"'
        assert bos.next_term.next_term.next_term.next_term.lookback == 1
        assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == 'c'
        assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2
        assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
        assert bos.next_term.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"'

        self.treemanager.key_normal("\"")

        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '""'
        assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"'
        assert bos.next_term.next_term.next_term.next_term.lookback == 1
        assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '"c"'
        assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2
        assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'


        self.treemanager.key_normal("\"")
        #assert self.parser.last_status == False


        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""'
        assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c'
        assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
        assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"'

        self.treemanager.key_end()
        self.treemanager.key_backspace()

        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""'
        assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c'
        assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'

    def test_ignore_nonlbox_x80(self):

        self.reset()
        inputstring = 'x="""ab\x80c"""'

        for i in inputstring:
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""ab\x80c"""'

    def test_multinode_from_the_start(self):

        self.reset()
        inputstring = '''x="""a\rbc"""'''

        for i in inputstring:
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'

    def test_multinode_and_nonlbox_x80(self):

        self.reset()
        inputstring = '''x="""a\x80bc"""'''

        for i in inputstring:
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""'

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")
        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'

    def test_multinode_nonlbox_and_lbox(self):
        self.reset()
        inputstring = '''x="""a\x80bc"""'''

        for i in inputstring:
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""'

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.add_languagebox(lang_dict["SQL"])
        self.treemanager.key_normal("S")
        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "x"
        assert bos.next_term.next_term.symbol.name == "="
        assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'
        multi = bos.next_term.next_term.next_term
        assert multi.children[0].symbol.name == "\"\"\"a\x80b"
        assert type(multi.children[1].symbol) is MagicTerminal
        assert multi.children[2].symbol.name == "c\"\"\""

    def test_multinode_merged_first(self):
        self.reset()
        inputstring = '''"""a\rbc"""'''

        for i in inputstring:
            self.treemanager.key_normal(i)

        for i in 'def"""':
            self.treemanager.key_normal(i)

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.symbol.name == "<Multinode>"
        assert bos.next_term.next_term.symbol.name == 'def'
        assert bos.next_term.next_term.next_term.symbol.name == '""'
        assert bos.next_term.next_term.next_term.next_term.symbol.name == '"'

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_backspace()
        self.treemanager.key_backspace()
        self.treemanager.key_backspace()

        assert bos.next_term.symbol.name == "<Multinode>"
        assert bos.next_term.next_term.symbol.name == "NEWLINE"
        assert bos.next_term.next_term.next_term.symbol.name == "eos"

    def test_multinode_string_bug(self):
        self.reset()
        inputstring = '''x="abc"'''
        for i in inputstring:
            self.treemanager.key_normal(i)

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.add_languagebox(lang_dict["SQL"])
        self.treemanager.key_normal("x")

        bos = self.parser.previous_version.parent.children[0]
        x = bos.next_term
        assert x.symbol.name == "x"
        eq = x.next_term
        assert eq.symbol.name == "="
        multi = eq.next_term
        assert multi.lookup == "dstring"
        assert multi.symbol.name == "<Multinode>"

        self.treemanager.cursor_movement(RIGHT)
        self.treemanager.key_backspace()

        # removing the ending quote results in a lexingerror,
        # so the multinode remains
        assert eq.next_term.symbol.name == "<Multinode>"

        # now remove the first quote, which should lead to the destruction of
        # the multinode
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)
        self.treemanager.key_backspace()
        assert eq.next_term.symbol.name == "abc"

    def test_multinode_string_bug2(self):
        self.reset()
        inputstring = '''x="abc"'''
        for i in inputstring:
            self.treemanager.key_normal(i)

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.add_languagebox(lang_dict["SQL"])
        self.treemanager.key_normal("x")

        self.treemanager.leave_languagebox()
        self.treemanager.key_normal("z")
        bos = self.parser.previous_version.parent.children[0]
        x = bos.next_term
        assert x.symbol.name == "x"
        eq = x.next_term
        assert eq.symbol.name == "="
        multi = eq.next_term
        assert multi.children[0].symbol.name == "\"abc"
        assert multi.children[1].symbol.name == "<SQL>"
        assert multi.children[2].symbol.name == "z\""
예제 #7
0
class Test_MultiTextNode:

    def setup_class(cls):
        grm = EcoFile("MultiTest", "test/calcmultistring.eco", "Multi")
        parser, lexer = grm.load()
        cls.lexer = lexer
        cls.parser = parser
        cls.parser.init_ast()
        cls.ast = cls.parser.previous_version
        cls.treemanager = TreeManager()
        cls.treemanager.add_parser(cls.parser, cls.lexer, calc.name)

        cls.treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite

    def reset(self):
        self.parser.reset()
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, calc.name)
        self.treemanager.set_font_test(7, 17)

    def test_simple(self):
        self.reset()
        self.treemanager.key_normal("1")
        self.treemanager.key_normal("+")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("abc")
        assert self.parser.last_status == False

        self.treemanager.key_normal("\"")
        assert self.parser.last_status == True

    def test_newline(self):
        self.reset()
        self.treemanager.key_normal("1")
        self.treemanager.key_normal("+")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("abc")
        self.treemanager.key_normal("\"")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

        assert self.parser.last_status == True

    def test_doublenewline(self):
        self.reset()
        self.treemanager.key_normal("1")
        self.treemanager.key_normal("+")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("abcd")
        self.treemanager.key_normal("\"")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

        assert self.parser.last_status == True

    def test_doublenewline_delete(self):
        self.reset()
        self.treemanager.key_normal("1")
        self.treemanager.key_normal("+")
        self.treemanager.key_normal("\"")
        self.treemanager.key_normal("abcd")
        self.treemanager.key_normal("\"")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

        assert self.parser.last_status == True

        self.treemanager.cursor_movement(LEFT)
        self.treemanager.cursor_movement(LEFT)

        self.treemanager.key_normal("\r")

        bos = self.parser.previous_version.parent.children[0]
        assert bos.next_term.next_term.next_term.children[0].symbol.name == "\"ab"
        assert bos.next_term.next_term.next_term.children[1].symbol.name == "\r"
        assert bos.next_term.next_term.next_term.children[2].symbol.name == "c"
        assert bos.next_term.next_term.next_term.children[3].symbol.name == "\r"
        assert bos.next_term.next_term.next_term.children[4].symbol.name == "d\""

        self.treemanager.cursor_movement(DOWN)
        self.treemanager.key_backspace()

        assert bos.next_term.symbol.name == "1"
        assert bos.next_term.next_term.symbol.name == "+"
        assert bos.next_term.next_term.next_term.children[0].symbol.name == "\"ab"
        assert bos.next_term.next_term.next_term.children[1].symbol.name == "\r"
        assert bos.next_term.next_term.next_term.children[2].symbol.name == "cd\""
        assert len(bos.next_term.next_term.next_term.children) == 3
        assert bos.next_term.next_term.next_term.children[2].next_term is None

        assert self.parser.last_status == True
예제 #8
0
class Test_Bootstrapping(object):
    def test_error(self):
        bootstrap = BootstrapParser()
        test_grammar = """S ::= A {If(child1=#1, child2=[#3, #4]}; %% a:\"a\""""
        py.test.raises(Exception, "bootstrap.parse(test_grammar)")

    def test_simple(self):
        bootstrap = BootstrapParser()
        test_grammar = """S ::= A {If(child1=#1, child2=[#3, #4])}; A ::= \"a\"; %% a:\"a\""""
        bootstrap.parse(test_grammar)

    def test_bootstrapping1(self):
        bootstrap = BootstrapParser(lr_type=1, whitespaces=False)
        test_grammar = 'S ::= "abc"; %% abc:\"abc\"'
        bootstrap.parse(test_grammar)
        self.treemanager = TreeManager()
        self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer,
                                    "")
        self.treemanager.set_font_test(7, 17)
        self.treemanager.key_normal("a")
        assert bootstrap.incparser.last_status == False
        self.treemanager.key_normal("b")
        assert bootstrap.incparser.last_status == False
        self.treemanager.key_normal("c")
        assert bootstrap.incparser.last_status == True

    def test_calculator(self):
        calc = """
            E ::= T             {#0}
                | E "plus" T    {Plus(arg1=#0,arg2=#2)}
                ;
            T ::= P             {#0}
                | T "mul" P     {Mul(arg1=#0,arg2=#2)}
                ;
            P ::= "INT"         {#0}
                ;
        %%

            INT:"[0-9]+"
            plus:"\+"
            mul:"\*"
            <ws>:"[ \\t]+"
            <return>:"[\\n\\r]"
        """
        bootstrap = BootstrapParser(lr_type=1, whitespaces=False)
        bootstrap.parse(calc)
        self.treemanager = TreeManager()
        self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer,
                                    "")
        self.treemanager.set_font_test(7, 17)
        self.treemanager.key_normal("1")
        assert bootstrap.incparser.last_status == True
        self.treemanager.key_normal("+")
        assert bootstrap.incparser.last_status == False
        self.treemanager.key_normal("2")
        assert bootstrap.incparser.last_status == True
        self.treemanager.key_normal("*")
        assert bootstrap.incparser.last_status == False
        self.treemanager.key_normal("3")
        assert bootstrap.incparser.last_status == True

        # test ast generation
        root = bootstrap.incparser.previous_version.parent
        assert root.symbol.name == "Root"
        assert root.children[1].symbol.name == "E"
        E = root.children[1]
        assert isinstance(E.alternate, AstNode)
        assert E.alternate.name == "Plus"
        plus = E.alternate
        assert plus.children['arg1'].symbol.name == "1"
        mul = plus.children['arg2']
        assert isinstance(mul, AstNode)
        assert mul.name == "Mul"
        assert mul.children['arg1'].symbol.name == "2"
        assert mul.children['arg2'].symbol.name == "3"

    def test_foreach(self):
        grammar = """
X ::= items {foreach(#0) Field(b=item.x)}
    ;

items ::= items "comma" item    {#0 + [#2]}
        | item                  {[#0]}
        ;

item ::= "a" {Var(x=#0)}
       | "b" {Var(x=#0)}
        ;

%%

a:"a"
b:"b"
comma:","

"""
        bootstrap = BootstrapParser(lr_type=1, whitespaces=False)
        bootstrap.parse(grammar)
        self.treemanager = TreeManager()
        self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer,
                                    "")
        self.treemanager.set_font_test(7, 17)
        self.treemanager.key_normal("a")
        assert bootstrap.incparser.last_status == True
        self.treemanager.key_normal(",")
        assert bootstrap.incparser.last_status == False
        self.treemanager.key_normal("b")
        assert bootstrap.incparser.last_status == True

        # test ast generation
        root = bootstrap.incparser.previous_version.parent
        assert root.symbol.name == "Root"
        assert root.children[1].symbol.name == "X"
        E = root.children[1]
        assert isinstance(E.alternate, ListNode)
        assert isinstance(E.alternate.children[0], AstNode)
        assert isinstance(E.alternate.children[1], AstNode)
        assert E.alternate.children[0].name == "Field"
        assert E.alternate.children[1].name == "Field"
        assert E.alternate.children[0].children['b'].symbol.name == "a"
        assert E.alternate.children[1].children['b'].symbol.name == "b"

    def test_lookup_reference(self):
        grammar = """
X ::= item {#0.x}
    ;

item ::= "a" {Var(x=#0)}
       | "b" {Var(x=#0)}
        ;

%%

a:"a"
b:"b"
comma:","

"""
        bootstrap = BootstrapParser(lr_type=1, whitespaces=False)
        bootstrap.parse(grammar)
        self.treemanager = TreeManager()
        self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer,
                                    "")
        self.treemanager.set_font_test(7, 17)
        self.treemanager.key_normal("a")
        assert bootstrap.incparser.last_status == True

        # test ast generation
        root = bootstrap.incparser.previous_version.parent
        assert root.symbol.name == "Root"
        assert root.children[1].symbol.name == "X"
        E = root.children[1]
        assert E.alternate.symbol.name == "a"
예제 #9
0
파일: ast_viewer.py 프로젝트: Britefury/eco
from viewer import Viewer
from grammars.grammars import python275_annotated
from treemanager import TreeManager
from incparser.incparser import IncParser
from inclexer.inclexer import IncrementalLexer
from incparser.astree import BOS, EOS

grammar = python275_annotated
whitespace = True

lexer = IncrementalLexer(grammar.priorities)
parser = IncParser(grammar.grammar, 1, whitespace)
parser.init_ast()
ast = parser.previous_version
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, grammar.name)
treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite

inputstring = """import abc.xyz as efg
from x import z

class Test:
    def x():
        if x == 1:
            z = 3 + 4 * 5
        elif x == 2:
            for x in range(2,10):
                print x
        else:
            z = 4
        return 1 if a==b else 2"""
예제 #10
0
class FuzzyLboxStats:
    def __init__(self, main, sub):
        parser, lexer = main.load()
        self.lexer = lexer
        self.parser = parser
        self.ast = parser.previous_version
        self.treemanager = TreeManager()
        self.treemanager.add_parser(parser, lexer, main.name)

        parser.setup_autolbox(main.name)
        self.sub = sub

        self.inserted = 0

    def load_main(self, filename):
        f = open(filename, "r")
        content = f.read()
        f.close()
        self.treemanager.import_file(content)
        self.mainexprs = self.find_nonterms_by_name(self.treemanager,
                                                    self.main_repl_str)

    def load_expr(self, filename):
        f = open(filename, "r")
        content = f.read()
        f.close()

        self.replexprs = self.find_expressions(content, self.sub_repl_str)

    def set_replace(self, main, sub):
        self.main_repl_str = main
        self.sub_repl_str = sub

    def find_nonterms_by_name(self, tm, name):
        l = []
        bos = tm.get_bos()
        eos = tm.get_eos()
        node = bos.right_sibling()
        while node is not eos:
            if node.symbol.name == name:
                l.append(node)
            if node.children:
                node = node.children[0]
                continue
            node = next_node(node)
        return l

    def find_expressions(self, program, expr):
        parser, lexer = self.sub.load()
        treemanager = TreeManager()
        treemanager.add_parser(parser, lexer, self.sub.name)
        treemanager.import_file(program)

        # find all sub expressions
        l = self.find_nonterms_by_name(treemanager, expr)
        return [subtree_to_text(st).rstrip() for st in l]

    def insert_python_expression(self, expr):
        for c in expr:
            self.treemanager.key_normal(c)

    def delete_expr(self, expr):
        # find first term and last term
        # select + delete
        node = expr
        while type(node.symbol) is Nonterminal:
            if node.children:
                node = node.children[0]
            else:
                node = next_node(node)
        first = node

        node = expr
        while type(node.symbol) is Nonterminal:
            if node.children:
                node = node.children[-1]
            else:
                node = prev_node(node)
        last = node

        if first.deleted or last.deleted:
            return None

        self.treemanager.select_nodes(first, last)
        deleted = self.treemanager.copySelection()
        self.treemanager.deleteSelection()
        return deleted

    def run(self):
        assert len(self.treemanager.parsers) == 1

        print self.main_repl_str, len(
            [subtree_to_text(x) for x in self.mainexprs])
        print self.sub_repl_str, len(self.replexprs)
        random.shuffle(self.mainexprs)
        for e in self.mainexprs[:10]:
            if e.get_root() is None:
                continue
            deleted = self.delete_expr(e)
            before = len(self.treemanager.parsers)
            if deleted:
                choice = random.choice(self.replexprs)
                print "  Replacing '{}' with '{}':".format(
                    truncate(deleted), truncate(choice))
                self.insert_python_expression(choice)
                valid = self.parser.last_status
                if before == len(self.treemanager.parsers):
                    result = "No box inserted"
                else:
                    result = "Box inserted"
                    self.inserted += 1
                print "    => {} ({})".format(result, valid)
            else:
                print "Replacing '{}' with '{}':\n    => Already deleted".format(
                    truncate(subtree_to_text(e)), truncate(choice))
        print("Boxes inserted: {}/{}".format(self.inserted, 10))
예제 #11
0
파일: bootstrap.py 프로젝트: Britefury/eco
class BootstrapParser(object):

    def __init__(self, lr_type=1, whitespaces=False):
        self.lr_type = lr_type
        self.whitespaces = whitespaces
        # load (old) parser for grammar grammar
        self.rules = {}
        self.lrules = []
        self.start_symbol = None
        self.incparser = None
        self.inclexer = None
        self.terminals = set()
        self.extra_alternatives = {}
        self.change_startrule = None
        self.options = {}
        self.precedences = []

    def implicit_ws(self):
        if self.options.has_key("implicit_ws"):
            if self.options["implicit_ws"] == "true":
                return True
        return False

    def indentation_based(self):
        if self.options.has_key("indentation"):
            if self.options["indentation"] == "true":
                return True
        return False

    def parse(self, ecogrammar):
        # this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco))
        from grammars.eco_grammar import eco_grammar as grammar
        self.lexer = IncrementalLexer(grammar.priorities)
        self.parser = IncParser(grammar.grammar, 1, True)
        self.parser.init_ast()
        self.ast = self.parser.previous_version.parent
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, grammar.name)
        self.treemanager.import_file(ecogrammar)
        if self.parser.last_status == False:
            raise Exception("Invalid input grammar: at %s %s" % (self.parser.error_node.prev_term, self.parser.error_node))
        self.read_options()
        self.create_parser()
        self.create_lexer()

    def read_options(self):
        startrule = self.ast.children[1] # startrule
        assert startrule.symbol.name == "Startrule"
        grammar = startrule.children[1]
        assert grammar.symbol.name == "grammar"
        for element in grammar.children:
            if element.symbol.name == "options":
                break
        if element.symbol.name != "options":
            # grammar has no options
            print("warning: grammar has no options")
            # backwards compatibility
            if self.whitespaces:
                self.options["implicit_ws"] = "true"
            return
        options = element
        assert options.symbol.name == "options"
        self.parse_options(options)

    def parse_options(self, options):
        if options.children == []:
            return
        if len(options.children) > 0:
            assert options.children[0].symbol.name == "settings"
            self.parse_settings(options.children[0])
        if len(options.children) > 1:
            assert options.children[1].symbol.name == "precedences"
            self.parse_precedences(options.children[1])

    def parse_settings(self, options):
        if options.children == []:
            return
        if len(options.children) == 2:
            more = options.children[0]
            self.parse_settings(more)
            option = options.children[1]
        else:
            option = options.children[0]
        name = option.children[2].symbol.name
        choice = option.children[6]
        assert choice.symbol.name == "choice"
        self.options[name] = choice.children[0].symbol.name

    def parse_precedences(self, precedences):
        if precedences.children == []:
            return
        # recursively parse other precedences
        if len(precedences.children) == 2:
            more = precedences.children[0]
            self.parse_precedences(more)
            precedence = precedences.children[1]
        else:
            precedence = precedences.children[0]
        # parse single precedence
        name = precedence.children[0].symbol.name
        terminals = self.parse_precedence_symbols(precedence.children[2])
        self.precedences.append((name, terminals))

    def parse_precedence_symbols(self, symbol):
        s = []
        for c in symbol.children:
            if c.symbol.name == "WS":
                continue
            if c.symbol.name == "terminals":
                rec_s = self.parse_precedence_symbols(symbol.children[0])
                s.extend(rec_s)
            if c.lookup == "terminal":
                s.append(c.symbol.name[1:-1])
        return s

    def create_parser(self, pickle_id = None):
        startrule = self.ast.children[1] # startrule
        grammar = startrule.children[1]
        parser = grammar.children[0]
        assert parser.symbol.name == "parser"
        self.parse_rules(parser)

        if self.implicit_ws():
            ws_rule = Rule()
            ws_rule.symbol = Nonterminal("WS")
            ws_rule.add_alternative([Terminal("<ws>"), Nonterminal("WS")])
            ws_rule.add_alternative([Terminal("<return>"), Nonterminal("WS")])
            ws_rule.add_alternative([Terminal("<backslash>"), Terminal("<return>"), Nonterminal("WS")])
            ws_rule.add_alternative([]) # or empty
            self.rules[ws_rule.symbol] = ws_rule

            # allow whitespace/comments at beginning of file
            start_rule = Rule()
            start_rule.symbol = Nonterminal("Startrule")
            start_rule.add_alternative([Nonterminal("WS"), self.start_symbol])
            self.rules[start_rule.symbol] = start_rule
            self.start_symbol = start_rule.symbol

        incparser = IncParser()
        incparser.from_dict(self.rules, self.start_symbol, self.lr_type, self.implicit_ws(), pickle_id, self.precedences)
        incparser.init_ast()
        self.incparser = incparser

    def parse_rules(self, node):
        if node.children[0].symbol.name == "parser":
            self.parse_rules(node.children[0])
            self.parse_rule(node.children[3])
        elif node.children[0].symbol.name == "rule":
            self.parse_rule(node.children[0])

    def parse_rule(self, node):
        name = node.children[0].symbol.name
        alternatives = self.parse_alternatives(node.children[4])
        symbol = Nonterminal(name)
        if self.start_symbol is None:
            self.start_symbol = symbol
        if self.change_startrule and symbol.name == self.change_startrule:
            self.start_symbol = symbol
        r = Rule(symbol)
        for a in alternatives:
            r.add_alternative(a[0], a[1], a[2])
        # add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes)
        if self.extra_alternatives.has_key(symbol.name):
            for n in self.extra_alternatives[symbol.name]:
                r.add_alternative([MagicTerminal(n), Nonterminal("WS")], None)
        self.rules[symbol] = r

    def parse_alternatives(self, node):
        if node.children[0].symbol.name == "alternatives":
            alternatives = self.parse_alternatives(node.children[0])
            alternative = self.parse_alternative(node.children[3])
            alternatives.append(alternative)
            return alternatives
        elif node.children[0].symbol.name == "right":
            return [self.parse_alternative(node.children[0])]

    def parse_alternative(self, node):
        if len(node.children) > 0:
            annotation = None
            prec = None
            for c in node.children:
                if c.symbol.name == "symbols":
                    symbols = self.parse_symbols(c)
                if c.symbol.name == "prec":
                    prec = self.parse_prec(c)
                if c.symbol.name == "annotations":
                    annotation = self.parse_annotation(c)
            return (symbols, annotation, prec)
        else:
            return ([], None, None)

    def parse_prec(self, node):
        if node.children:
            c = node.children[2]
            return c.symbol.name[1:-1]

    def parse_symbols(self, node):
        if node.children[0].symbol.name == "symbols":
            symbols = self.parse_symbols(node.children[0])
            symbol = self.parse_symbol(node.children[1])
            symbols.append(symbol)
            if (isinstance(symbol, Terminal) or isinstance(symbol, MagicTerminal)) and self.implicit_ws():
                symbols.append(Nonterminal("WS"))
            return symbols
        elif node.children[0].symbol.name == "symbol":
            l = []
            symbol = self.parse_symbol(node.children[0])
            l.append(symbol)
            if isinstance(symbol, Terminal) and self.implicit_ws():
                l.append(Nonterminal("WS"))
            return l

    def parse_symbol(self, node):
        node = node.children[0]
        if node.lookup == "nonterminal":
            return Nonterminal(node.symbol.name)
        elif node.lookup == "terminal":
            self.terminals.add(node.symbol.name[1:-1])
            return Terminal(node.symbol.name[1:-1])
        elif node.lookup == "languagebox":
            return MagicTerminal(node.symbol.name)

    def parse_annotation(self, node):
        a_options = node.children[2]
        assert a_options.symbol.name == "a_options"
        if a_options.children[0].symbol.name == "astnode":
            return self.parse_astnode(a_options.children[0])
        elif a_options.children[0].symbol.name == "expression":
            return self.parse_expression(a_options.children[0])
        elif a_options.children[0].symbol.name == "forloop":
            return self.parse_foreach(a_options.children[0])

    def parse_astnode(self, node):
        name = node.children[0].symbol.name
        children = self.parse_astnode_children(node.children[4])
        d = {}
        for n, expr in children:
            d[n] = expr
        return AstNode(name, d)

    def parse_astnode_children(self, node):
        assert node.symbol.name == "astnode_children"
        if node.children[0].symbol.name == "astnode_child":
            return [self.parse_astnode_child(node.children[0])]
        elif node.children[0].symbol.name == "astnode_children":
            children = self.parse_astnode_children(node.children[0])
            child = self.parse_astnode_child(node.children[3])
            children.append(child)
            return children

    def parse_astnode_child(self, node):
        assert node.symbol.name == "astnode_child"
        name = node.children[0].symbol.name
        if node.children[4].symbol.name == "expression":
            expr = self.parse_expression(node.children[4])
        elif node.children[4].symbol.name == "reference":
            expr = self.parse_reference(node.children[4])
        return (name, expr)

    def parse_expression(self, node):
        if node.children[0].symbol.name == "node":
            return self.parse_node(node.children[0])
        elif node.children[0].symbol.name == "list":
            return self.parse_list(node.children[0])
        elif node.children[0].symbol.name == "node_ref":
            return self.parse_noderef(node.children[0])
        else:
            expr1 = self.parse_expression(node.children[0])
            if node.children[3].symbol.name == "node":
                expr2 = self.parse_node(node.children[3])
            else:
                expr2 = self.parse_list(node.children[3])
            return AddExpr(expr1, expr2)

    def parse_foreach(self, node):
        item = self.parse_node(node.children[4])
        expr = self.parse_astnode(node.children[7])
        return Foreach(node.symbol.name, item, expr)

    def parse_noderef(self, node):
        lookup = self.parse_node(node.children[0])
        attr = node.children[3]
        lookup.attribute = attr.symbol.name
        return lookup

    def parse_node(self, node):
        return LookupExpr(int(node.children[2].symbol.name))

    def parse_list(self, node):
        return ListExpr(self.parse_listloop(node.children[2]))

    def parse_reference(self, node):
        base = node.children[0].symbol.name
        ref = node.children[4].symbol.name
        return ReferenceExpr(base, ref)

    def parse_listloop(self, node):
        if len(node.children) == 0:
            return []
        if node.children[0].symbol.name == "list_loop":
            l = self.parse_listloop(node.children[0])
            element = self.parse_unknown(node.children[3])
            l.append(element)
            return l
        else:
            return [self.parse_unknown(node.children[0])]

    def parse_unknown(self, node):
        if node.symbol.name == "node":
            return self.parse_node(node)
        elif node.symbol.name == "astnode":
            return self.parse_astnode(node)

    def create_lexer(self):
        startrule = self.ast.children[1] # startrule
        grammar = startrule.children[1]
        for element in grammar.children:
            if element.symbol.name == "lexer":
                break
        lexer = element
        assert lexer.symbol.name == "lexer"
        self.parse_lexer(lexer)
        names = []
        regexs = []
        for name, regex in self.lrules:
            names.append(name)
            regexs.append(regex)
        # add so far undefined terminals
        undefined_terminals = self.terminals.difference(set(names))
        import re
        for t in undefined_terminals:
            names.insert(0, t)
            regexs.insert(0,re.escape(t))
        self.inclexer = IncrementalLexerCF()
        self.inclexer.from_name_and_regex(names, regexs)
        if self.indentation_based():
            self.inclexer.indentation_based = True

    def parse_lexer(self, lexer):
        if lexer.children[0].symbol.name == "lrule":
            self.parse_lrule(lexer.children[0])
        elif lexer.children[0].symbol.name == "lexer":
            self.parse_lexer(lexer.children[0])
            self.parse_lrule(lexer.children[1])

    def parse_lrule(self, lrule):
        assert lrule.children[0].symbol.name == "tokenname"
        name = lrule.children[0].children[0].symbol.name
        regex = lrule.children[3].symbol.name[1:-1]
        self.lrules.append((name, regex))
예제 #12
0
파일: nodeeditor.py 프로젝트: Britefury/eco
class NodeEditor(QFrame):

    # ========================== init stuff ========================== #

    def __init__(self, parent=None):
        QtGui.QWidget.__init__(self, parent)

        self.infofont = QtGui.QFont('Courier', 6)
        self.infofontht = QtGui.QFontMetrics(self.infofont).height() + 3
        self.infofontwt = QtGui.QFontMetrics(self.infofont).width(" ")

        self.viewport_y = 0 # top visible line
        self.imagemode = False
        self.image = None

        self.scroll_height = 0
        self.scroll_width = 0

        self.timer = QTimer(self)
        self.backuptimer = QTimer(self)
        self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer)
        self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer)
        self.backuptimer.start(30000)
        self.undotimer = QTimer(self)
        self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer)

        self.blinktimer = QTimer(self)
        self.blinktimer.start(500)
        self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer)
        self.show_cursor = True

        self.lightcolors = [QColor("#333333"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")]
        self.darkcolors = [QColor("#999999"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")]
        self.setCursor(Qt.IBeamCursor)

    def analysis_timer(self):
        if self.getWindow().show_namebinding():
            self.tm.analyse()
            self.update()
        self.timer.stop()

        # save swap
        filename = self.getEditorTab().filename
        if filename:
            self.saveToJson(filename + ".swp", True)

    def backup_timer(self):
        filename = self.getEditorTab().filename
        if filename:
            self.saveToJson(filename + ".bak", True)

    def trigger_blinktimer(self):
        if self.timer.isActive():
            self.show_cursor = True
            return
        self.show_cursor ^= True
        self.update()

    def trigger_undotimer(self):
        self.tm.save_current_version()
        self.undotimer.stop()

    def setImageMode(self, boolean):
        self.imagemode = boolean

    def reset(self):
        #self.getWindow().ui.scrollArea.horizontalScrollBar().setValue(0)
        #self.getWindow().ui.scrollArea.verticalScrollBar().setValue(0)
        self.update()

    def set_mainlanguage(self, parser, lexer, lang_name):
        self.tm = TreeManager()
        self.tm.add_parser(parser, lexer, lang_name)

    def set_sublanguage(self, language):
        self.sublanguage = language

    def event(self, event):
        if event.type() == QEvent.ToolTip:
            if QToolTip.isVisible():
                QToolTip.hideText()
                event.ignore()
                return True
            pos = event.pos()
            temp_cursor = self.tm.cursor.copy()
            result = self.coordinate_to_cursor(pos.x(), pos.y())
            node = self.tm.cursor.node
            self.tm.cursor.line = temp_cursor.line
            self.tm.cursor.node = temp_cursor.node
            self.tm.cursor.pos = temp_cursor.pos
            if not result:
                event.ignore()
                return True
            msg = self.tm.get_error(node)
            if not msg:
                msg = self.tm.get_error(node)
            if msg:
                QToolTip.showText(event.globalPos(), msg);
            return True
        return QFrame.event(self, event)

    # ========================== GUI related stuff ========================== #

    def blink(self):
        if self.show_cursor:
            self.show_cursor = 0
        else:
            self.show_cursor = 1
        self.update()

    def sliderChanged(self, value):
        change = self.viewport_y - value
        self.viewport_y = value
        self.update()

    def sliderXChanged(self, value):
        self.move(-value*self.fontwt,0)
        self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height())
        if self.x() == 0:
            self.updateGeometry()
        self.update()

    def getScrollSizes(self):
        total_lines = 0
        max_width = 0
        for l in self.lines:
            total_lines += l.height
            max_width = max(max_width, l.width)
        max_visible_lines = self.geometry().height() / self.fontht
        self.scroll_height = max(0, total_lines - max_visible_lines)

        current_width = self.parentWidget().geometry().width() / self.fontwt
        self.scroll_width = max(0, max_width - current_width)

    def paintEvent(self, event):
        gfont = QApplication.instance().gfont
        self.font = gfont.font
        self.fontwt = gfont.fontwt
        self.fontht = gfont.fontht
        QtGui.QFrame.paintEvent(self, event)
        paint = QtGui.QPainter()
        if self.imagemode:
            self.image = QImage()
            paint.begin(self.image)
        else:
            paint.begin(self)
        paint.setFont(self.font)

        y = 0
        x = 0

        self.longest_column = 0

        # calculate how many lines we need to show
        self.init_height = self.geometry().height()

        self.paintLines(paint, self.viewport_y)

        paint.end()

        total_lines = 0
        max_width = 0
        for l in self.lines:
            total_lines += l.height
            max_width = max(max_width, l.width)
        max_visible_lines = self.geometry().height() / self.fontht
        self.scroll_height = max(0, total_lines - max_visible_lines)


        current_width = self.parentWidget().geometry().width() / self.fontwt
        self.scroll_width = max(0, max_width - current_width)

        self.emit(SIGNAL("painted()"))

    # paint lines using new line manager
    def paintLines(self, paint, startline):

        # find internal line corresponding to visual line
        visual_line = 0
        internal_line = 0
        for l in self.tm.lines:
            if visual_line + l.height > startline:
                break
            visual_line += l.height
            internal_line += 1

        x = 0
        y = visual_line - startline # start drawing outside of viewport to display partial images
        self.paint_start = (internal_line, y)

        max_y = self.geometry().height() / self.fontht

        line = internal_line
        node = self.tm.lines[line].node

        self.paint_nodes(paint, node, x, y, line, max_y)


    #XXX if starting node is inside language box, init lbox with amout of languge boxes
    def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0):

        settings = QSettings("softdev", "Eco")
        if settings.value("app_theme", "Light").toString() == "Dark":
            alpha = 100
            colors = self.darkcolors
        else:
            alpha = 40
            colors = self.lightcolors

        first_node = node
        selected_language = self.tm.mainroot
        error_node = self.tm.get_mainparser().error_node
        error_node = self.fix_errornode(error_node)

        highlighter = self.get_highlighter(node)
        selection_start = min(self.tm.selection_start, self.tm.selection_end)
        selection_end = max(self.tm.selection_start, self.tm.selection_end)
        draw_selection_start = (0,0,0)
        draw_selection_end = (0,0,0)
        start_lbox = self.get_languagebox(node)
        editor = self.get_editor(node)

        self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node)
        #XXX get initial x for langbox

        if start_lbox:
            lbox += 1
        if start_lbox and self.selected_lbox is start_lbox:
            draw_lbox = True
        else:
            draw_lbox = False

        draw_all_boxes = self.getWindow().show_languageboxes()

        self.lines = self.tm.lines
        self.cursor = self.tm.cursor
        self.lines[line].height = 1 # reset height
        draw_cursor = True
        #l_x = [0]
        while y < max_y:

            # if we found a language box, continue drawing inside of it
            if isinstance(node.symbol, MagicTerminal):
                #l_x.append(x)
                #node.pos = x
                lbox += 1
                lbnode = node.symbol.ast
                if self.selected_lbox is node:
                    draw_lbox = True
                    selected_language = lbnode
                else:
                    draw_lbox = False
                node = lbnode.children[0]
                highlighter = self.get_highlighter(node)
                editor = self.get_editor(node)
                error_node = self.tm.get_parser(lbnode).error_node
                error_node = self.fix_errornode(error_node)
                continue

            if isinstance(node, EOS):
                lbnode = self.get_languagebox(node)
                if self.cursor.node is lbnode:
                    self.draw_cursor(paint, x, 5 + y * self.fontht)
                if lbnode:
                    #l_x.pop()
                    if lbox > 0:
                        lbox -= 1
                    node = lbnode.next_term
                    highlighter = self.get_highlighter(node)
                    editor = self.get_editor(node)
                    if self.selected_lbox is lbnode:
                        draw_lbox = False
                    lbnode = self.get_languagebox(node)
                    if lbnode and self.selected_lbox is lbnode:
                        draw_lbox = True
                        error_node = self.tm.get_parser(lbnode.symbol.ast).error_node
                        error_node = self.fix_errornode(error_node)
                    else:
                        error_node = self.tm.get_mainparser().error_node
                        error_node = self.fix_errornode(error_node)
                    continue
                else:
                    self.lines[line].width = x / self.fontwt
                    break

            # draw language boxes
            if lbox > 0 and (draw_lbox or draw_all_boxes):
                #color = self.nesting_colors[lbox % 5]
                if draw_all_boxes:
                    color = colors[(lbox-1) % len(colors)]
                    color.setAlpha(alpha)
                else:
                    color = colors[0]
                    color.setAlpha(alpha)
                if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box
                    color = colors[-1] # overwrite color
                    color.setAlpha(alpha)
                editor.update_image(node)
                if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal):
                    if not node.image or node.plain_mode:
                        paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color)

            # prepare selection drawing
            if node is selection_start.node:
                if node.lookup == "<return>":
                    sel_x = x
                else:
                    sel_x = x + selection_start.pos * self.fontwt
                draw_selection_start = (sel_x, y, line)

            if node is selection_end.node:
                draw_selection_end = (x + selection_end.pos * self.fontwt, y, line)

            # draw node
            dx, dy = editor.paint_node(paint, node, x, y, highlighter)
            x += dx
            #y += dy
            self.lines[line].height = max(self.lines[line].height, dy)

            # after we drew a return, update line information
            if node.lookup == "<return>" and not node is first_node:
                # draw lbox to end of line
                if draw_lbox or (draw_all_boxes and lbox > 0):
                    paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color)

                self.lines[line].width = x / self.fontwt
                x = 0#l_x[-1]
                y += self.lines[line].height
                line += 1
                self.lines[line].height = 1 # reset height

            # draw cursor
            if node is self.cursor.node and self.show_cursor:
                draw_x = max(0, x-dx)
                cursor_pos = self.cursor.pos

                if node.symbol.name == "\r":
                    cursor_pos = 0
                if node.image and not node.plain_mode:
                    draw_x = x
                    cursor_pos = 0
                self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 5 + y * self.fontht)


            if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor:
                draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3)
                paint.drawRect(draw_cursor_at)

                # set lbox info coordinates
                infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht)
                draw_cursor = False

            # draw squiggly line
            if node is error_node or (self.getWindow().show_namebinding() and self.tm.has_error(node)):
                if isinstance(node, EOS):
                    length = self.fontwt
                else:
                    length = len(node.symbol.name)*self.fontwt
                if isinstance(node.symbol, MagicTerminal):
                    self.draw_vertical_squiggly_line(paint,x,y)
                else:
                    if self.tm.has_error(node):
                        color = "orange"
                    else:
                        color = "red"
                    self.draw_squiggly_line(paint,x-length,y,length, color)

            node = node.next_term

        if selection_start != selection_end:
            self.draw_selection(paint, draw_selection_start, draw_selection_end)

        # paint infobox
        if False:
            lang_name = self.parser_langs[selected_language]
            lang_status = self.parsers[selected_language].last_status
            if lang_status is True:
                color = QColor(100,255,100)
            else:
                color = QColor(255,100,100)
            paint.setFont(self.infofont)
            paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*self.infofontwt, self.infofontht), color)
            paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name)
            paint.setFont(self.font)

        return x, y, line

    def fix_errornode(self, error_node):
        if not error_node:
            return
        while isinstance(error_node.symbol, IndentationTerminal):
            error_node = error_node.prev_term
        return error_node

    def draw_cursor(self, paint, x, y):
        pen = paint.pen()
        colorhex = self.palette().color(QPalette.Text)
        pen.setColor(QColor(colorhex))
        paint.setPen(pen)
        draw_cursor_at = QRect(x, y, 0, self.fontht - 3)
        paint.drawRect(draw_cursor_at)

    def draw_vertical_squiggly_line(self, paint, x, y):
        paint.setPen(Qt.CustomDashLine)
        pen = paint.pen()
        pen.setDashPattern([2,2])
        pen.setColor(QColor("red"))
        paint.setPen(pen)
        y = 3+y*self.fontht
        paint.drawLine(x-1, y, x-1, y+self.fontht)
        paint.drawLine(x, y+2, x, y+self.fontht)
        paint.setPen(Qt.SolidLine)

    def draw_squiggly_line(self, paint, x, y, length, color):
        paint.setPen(Qt.CustomDashLine)
        pen = paint.pen()
        pen.setDashPattern([2,2])
        pen.setColor(QColor(color))
        paint.setPen(pen)
        #x -= length
        y = (y+1)*self.fontht+1
        paint.drawLine(x, y, x+length, y)
        paint.drawLine(x+2, y+1, x+2+length, y+1)
        paint.setPen(Qt.SolidLine)

    def draw_selection(self, paint, draw_selection_start, draw_selection_end):
        x1, y1, line1 = draw_selection_start
        x2, y2, line2 = draw_selection_end
        if y1 == y2:
            paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100))
        else:
            paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, self.tm.lines[line1].width*self.fontwt - x1, self.fontht), QColor(0,0,255,100))
            y = y1 + self.tm.lines[line1].height
            for i in range(line1+1, line2):
                paint.fillRect(QRectF(0, 3 + y * self.fontht, self.tm.lines[i].width*self.fontwt, self.fontht), QColor(0,0,255,100))
                y = y + self.tm.lines[i].height
            paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100))

    def get_highlighter(self, node):
        root = node.get_root()
        base = lang_dict[self.tm.get_language(root)].base
        s = syntaxhighlighter.get_highlighter(base, self.palette())
        return s

    def get_languagebox(self, node):
        root = node.get_root()
        lbox = root.get_magicterminal()
        return lbox

    def get_editor(self, node):
        root = node.get_root()
        base = lang_dict[self.tm.get_language(root)].base
        return editor.get_editor(base, self.fontwt, self.fontht)

    def focusNextPrevChild(self, b):
        # don't switch to next widget on TAB
        return False

    def mousePressEvent(self, e):
        if e.button() == Qt.LeftButton:
            self.tm.input_log.append("# mousePressEvent")
            self.coordinate_to_cursor(e.x(), e.y())
           # self.tm.cursor = cursor
            self.tm.selection_start = self.tm.cursor.copy()
            self.tm.selection_end = self.tm.cursor.copy()
            self.tm.input_log.append("self.selection_start = self.cursor.copy()")
            self.tm.input_log.append("self.selection_end = self.cursor.copy()")
            #self.tm.fix_cursor_on_image()
            self.getWindow().showLookahead()
            self.update()

    def mouseDoubleClickEvent(self, e):
        if e.button() == Qt.LeftButton:
            self.coordinate_to_cursor(e.x(), e.y())
            node = self.tm.get_node_from_cursor()
            lbox = self.get_languagebox(node)
            if lbox and lbox.symbol.name == "<IPython>":
                if lbox.plain_mode is False:
                    lbox.plain_mode = True
                else:
                    lbox.plain_mode = False
                self.update()
                return
            elif node.image is None:
                self.tm.selection_start = self.tm.cursor.copy()
                self.tm.selection_start.node = self.tm.cursor.find_previous_visible(self.tm.cursor.node)
                self.tm.selection_start.pos = len(self.tm.selection_start.node.symbol.name)
                self.tm.selection_end = self.tm.cursor.copy()
                self.tm.selection_end.pos = len(self.tm.selection_end.node.symbol.name)
                self.tm.cursor.pos = self.tm.selection_end.pos
                self.tm.cursor.node = self.tm.selection_end.node
                self.update()
                return

            if node.plain_mode is False:
                node.plain_mode = True
                self.tm.cursor.pos = len(node.symbol.name)
            else:
                node.plain_mode = False
            self.update()

    def cursor_to_coordinate(self):
        y = 0
        for l in self.tm.lines[:self.cursor.line]:
            y += l.height * self.fontht
        x = self.tm.cursor.get_x() * self.fontwt
        y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht
        return (x,y)

    def coordinate_to_cursor(self, x, y):

        mouse_y = y / self.fontht
        first_line = self.paint_start[0]
        y_offset = self.paint_start[1]

        y = y_offset
        line = first_line
        while line < len(self.tm.lines) - 1:
            y += self.tm.lines[line].height
            if y > mouse_y:
                break
            line += 1

        self.tm.cursor.line = line
        cursor_x = int(round(float(x) / self.fontwt))
        self.tm.cursor.move_to_x(cursor_x, self.tm.lines)

        self.tm.input_log.append("self.cursor.line = %s" % str(line))
        self.tm.log_input("cursor.move_to_x", str(cursor_x), "self.lines")

        if mouse_y > y or self.tm.cursor.get_x() != cursor_x:
            return False
        return True

    def mouseMoveEvent(self, e):
        # apparaently this is only called when a mouse button is clicked while
        # the mouse is moving
        if self.tm.input_log[-2].startswith("self.cursor.move_to_x"):
            # only log the last move event
            self.tm.input_log.pop()
            self.tm.input_log.pop()
            self.tm.input_log.pop()
        self.coordinate_to_cursor(e.x(), e.y())
        self.tm.selection_end = self.tm.cursor.copy()
        self.tm.input_log.append("self.selection_end = self.cursor.copy()")
        self.update()

    def key_to_string(self, key):
        if key == Qt.Key_Up:
            return "up"
        if key == Qt.Key_Down:
            return "down"
        if key == Qt.Key_Left:
            return "left"
        if key == Qt.Key_Right:
            return "right"

    def keyPressEvent(self, e):

        startundotimer = False
        self.timer.start(500)
        self.show_cursor = True

        if e.key() in [Qt.Key_Shift, Qt.Key_Alt, Qt.Key_Control, Qt.Key_Meta, Qt.Key_AltGr]:
            if e.key() == Qt.Key_Shift:
                self.tm.key_shift()
            return

        text = e.text()

        self.edit_rightnode = False # has been processes in get_nodes_at_pos -> reset

        if e.key() == Qt.Key_Escape:
            self.tm.key_escape()
        elif e.key() == Qt.Key_Backspace:
            startundotimer = True
            self.tm.key_backspace()
        elif e.key() in [Qt.Key_Up, Qt.Key_Down, Qt.Key_Left, Qt.Key_Right]:
            if e.modifiers() == Qt.ShiftModifier:
                self.tm.key_cursors(self.key_to_string(e.key()), True)
            elif e.modifiers() == Qt.ControlModifier:
                self.tm.ctrl_cursor(self.key_to_string(e.key()))
            else:
                self.tm.key_cursors(self.key_to_string(e.key()), False)
        elif e.key() == Qt.Key_Home:
            self.tm.key_home(e.modifiers() == Qt.ShiftModifier)
        elif e.key() == Qt.Key_End:
            self.tm.key_end(e.modifiers() == Qt.ShiftModifier)
        elif e.key() == Qt.Key_Delete:
            startundotimer = True
            self.tm.key_delete()
        elif e.key() == Qt.Key_F3:
            self.tm.find_next()
        elif e.key() in [Qt.Key_PageUp, Qt.Key_PageDown]:
            pass # ignore those keys
        elif e.modifiers() in [Qt.ControlModifier, Qt.AltModifier]:
            # User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't
            # sensibly insert into the text.
            pass
        else:
            startundotimer = True
            if e.key() == Qt.Key_Tab:
                text = "    "
            else:
                text = e.text()
                if text.toUtf8() not in whitelist:
                    logging.debug("Key %s not supported" % text)
                    return
            self.tm.key_normal(text)

        self.getWindow().btReparse([])
        self.update()
        self.emit(SIGNAL("keypress(QKeyEvent)"), e)
        self.getWindow().showLookahead()
        if startundotimer:
            self.undotimer.start(500)

    def showLanguageBoxMenu(self):
        self.showSubgrammarMenu()
        self.create_languagebox()

    def create_languagebox(self):
        if self.sublanguage:
            if self.tm.hasSelection():
                self.tm.surround_with_languagebox(self.sublanguage)
            else:
                self.tm.add_languagebox(self.sublanguage)

    def change_languagebox(self):
        if self.sublanguage:
            self.tm.change_languagebox(self.sublanguage)

    def showCodeCompletion(self):
        l = self.tm.getCompletion()
        if l:
            self.showCodeCompletionMenu(l)

    def println(self, prestring, y):
        node = self.lines[y].node.next_term
        x = []
        while node is not None and node.symbol.name != "\r":
            x.append(node.symbol.name)
            node = node.next_term
        print(prestring, "".join(x))

    def print_line(self, y):
        current = self.lines[y].node
        while True:
            print(current)
            current = current.next_term
            if current is None:
                return

    # ========================== AST modification stuff ========================== #

    def insertTextNoSim(self, text):
        self.viewport_y = 0
        self.tm.import_file(text)
        return

    def getTL(self):
        return self.getWindow().tl

    def getPL(self):
        return self.getWindow().pl

    def getLRP(self):
        return self.getWindow().lrp

    def getWindow(self):
        return self.window()

    def getEditorTab(self):
        return self.parent().parent().parent()

    def getScrollArea(self):
        return self.parent().parent()

    def createSubgrammarMenu(self, menu, change=False):
        self.sublanguage = None

        tmp = None
        if change:
            # try and find lbox and set cursor to previous node before getting
            # lookahead list
            root = self.tm.cursor.node.get_root()
            lbox = root.get_magicterminal()
            if lbox:
                tmp = self.tm.cursor.node
                self.tm.cursor.node = lbox.prev_term

        # Create actions
        bf = QFont()
        bf.setBold(True)
        valid_langs = []
        for l in languages:
            if "<%s>" % l in self.tm.getLookaheadList():
                valid_langs.append(l)

        if tmp:
            # undo cursor change
            self.tm.cursor.node = tmp
        if len(valid_langs) > 0:
            for l in valid_langs:
                item = QAction(str(l), menu)
                item.setData(l)
                self._set_icon(item, l)
                item.setFont(bf)
                menu.addAction(item)
            menu.addSeparator()
        for l in languages:
            item = QAction(str(l), menu)
            item.setData(l)
            self._set_icon(item, l)
            menu.addAction(item)
        return menu

    def showSubgrammarMenu(self):
        menu = QtGui.QMenu("Language", self)
        self.createSubgrammarMenu(menu)
        x,y = self.cursor_to_coordinate()
        action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
        if action:
            self.sublanguage = action.data().toPyObject()
            self.edit_rightnode = True

    def _set_icon(self, mitem, lang):
        if lang.base.lower() == "html":
            icon = QIcon.fromTheme("text-xhtml+xml")
        else:
            icon = QIcon.fromTheme("text-x-" + lang.base.lower())
        if icon.isNull():
            icon = QIcon.fromTheme("application-x-" + lang.base.lower())
            if icon.isNull():
                icon = QIcon.fromTheme("text-x-generic")
        mitem.setIcon(icon)

    def showCodeCompletionMenu(self, l):
        menu = QtGui.QMenu( self )
        # Create actions
        toolbar = QtGui.QToolBar()
        for n in l:
            path = []
            for p in n.path:
                if p and p.name:
                    path.append(p.name)
            if n.vartype:
                vartype = n.vartype
                while vartype.children != []:
                    try:
                        vartype = vartype.children[0]
                    except KeyError:
                        vartype = vartype.children["name"]
                text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind)
            elif n.kind == "method":
                text = self.cc_method(n) + " - %s" % (".".join(path))
            else:
                text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind)
            item = toolbar.addAction(text, self.createCCFunc(n.name))
            item.setIcon(QIcon("gui/" + n.kind + ".png"))
            menu.addAction(item)
        x,y = self.cursor_to_coordinate()
        menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))

    def cc_method(self, n):
        s = [n.name, "("]
        param_ln = n.astnode.children["params"]
        if isinstance(param_ln, ListNode):
            for p in param_ln.children:
                tmp = p.children["type"]
                if isinstance(tmp, AstNode):
                    s.append(tmp.children["name"].symbol.name)
                else:
                    s.append(tmp.symbol.name)
                s.append(" ")
                s.append(p.children["name"].symbol.name)
                if p != param_ln.children[-1]:
                    s.append(", ")
        s.append(")")
        return "".join(s)

    def createMenuFunction(self, l):
        def action():
            self.sublanguage = l
            self.edit_rightnode = True
        return action

    def createCCFunc(self, text):
        def action():
            self.tm.pasteCompletion(text)
        return action

    def selectSubgrammar(self, item):
        pass

    def saveToJson(self, filename, swap=False):
        whitespaces = self.tm.get_mainparser().whitespaces
        root = self.tm.parsers[0][0].previous_version.parent
        language = self.tm.parsers[0][2]
        manager = JsonManager()
        manager.save(root, language, whitespaces, filename)
        if not swap:
            self.tm.changed = False
            self.emit(SIGNAL("painted()"))

    def loadFromJson(self, filename):
        manager = JsonManager()
        language_boxes = manager.load(filename)

        self.tm = TreeManager()

        self.tm.load_file(language_boxes)
        self.reset()

    def export(self, run=False):
        return self.tm.export(None, run)
예제 #13
0
    def random_deletion(self):
        """Delete random characters within a program."""
        print "Running random_deletion on {}".format(self.filename)
        program = self.program

        self.treemanager.import_file(program)
        assert self.parser.last_status == True

        self.text_compare(program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)
        random_lines = random_lines[:
                                    20]  # restrict to 20 lines to reduce runtime

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.treemanager.cursor_reset()
                self.log.append("self.treemanager.cursor_reset()")
                self.move(DOWN, linenr)
                self.log.append("self.move(DOWN, %s)" % linenr)
                self.move(RIGHT, col)
                self.log.append("self.move(RIGHT, %s)" % col)
                self.log.append("self.treemanager.key_delete()")
                x = self.treemanager.key_delete()
                if x == "eos":
                    continue
            self.treemanager.undo_snapshot()
            self.log.append("self.treemanager.undo_snapshot()")

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        parser.init_ast()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.set_font_test(7, 17)
        t1.import_file(self.program)

        assert self.parser.last_status == True
        assert parser.last_status == True

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)
예제 #14
0
    def random_insertdelete(self):
        """Insert and delete random characters at random locations within a
        program."""
        print "Running random_insertdelete on {}".format(self.filename)
        self.reset()

        self.treemanager.import_file(self.program)
        assert self.parser.last_status == True

        self.text_compare(self.program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)
        random_lines = random_lines[:
                                    20]  # restrict to 20 lines to reduce runtime

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.log.append("self.treemanager.cursor_reset()")
                self.log.append("self.move(%s, %s)" % (DOWN, linenr))
                self.log.append("self.move(%s, %s)" % (RIGHT, col))
                self.treemanager.cursor_reset()
                self.move(DOWN, linenr)
                self.move(RIGHT, col)
                k = self.get_random_key()
                if k in [
                        "a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7"
                ]:
                    # for a few characters DELETE instead of INSERT
                    self.log.append("self.treemanager.key_delete()")
                    x = self.treemanager.key_delete()
                else:
                    rk = self.get_random_key()
                    self.log.append("self.treemanager.key_normal(%s)" % rk)
                    x = self.treemanager.key_normal(rk)
                if x == "eos":
                    continue
            self.log.append("self.treemanager.undo_snapshot()")
            self.treemanager.undo_snapshot()

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.import_file(self.program)

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)
예제 #15
0
class FuzzyTester():
    """Runs test that randomly modify a given program and tests for exceptions.
    If an exception occurs, or the modifications had unwanted side-effects, a
    log is saved which can be used to create a stand-alone test."""
    def __init__(self, filename):
        _, ext = os.path.splitext(filename)
        self.setlang(ext_to_lang[ext])
        with open(filename) as f:
            self.program = f.read()
        self.log = []
        self.filename = filename

    def reset(self):
        self.parser.reset()
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, self.lang.name)

    def setlang(self, lang):
        self.lang = lang_dict[lang]
        self.parser, self.lexer = lang_dict[lang].load()
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, self.lang.name)

    def move(self, direction, times):
        for i in range(times):
            self.treemanager.cursor_movement(direction)

    def text_compare(self, original):
        """Compare the textual representation of two programs. Faster than
        tree_compare, but less accurate."""
        original = original.replace("\r", "").split("\n")
        current = self.treemanager.export_as_text("/dev/null").replace(
            "\r", "").split("\n")
        for i in xrange(len(current)):
            assert original[i] == current[i]

    def next_node(self, node):
        if node.children:
            return node.children[0]
        while node.right_sibling() is None:
            node = node.parent
        return node.right_sibling()

    def tree_compare(self, node1, node2):
        """Given two root nodes, compares that both trees are equivalent"""
        while True:
            assert node1.symbol == node2.symbol
            if node1.right:
                assert node1.right.symbol == node2.right.symbol
            if node1.next_term:
                assert node1.next_term.symbol == node2.next_term.symbol
            if isinstance(node1.symbol, MagicTerminal):
                self.tree_compare(node1.symbol.ast, node2.symbol.ast)
            if isinstance(node1, EOS) and isinstance(node2, EOS):
                break
            node1 = self.next_node(node1)
            node2 = self.next_node(node2)

    def get_random_key(self):
        keys = list(
            "abcdefghijklmnopqrstuvwxyz0123456789 \r:,.[]{}()!$%^&*()_+=")
        return random.choice(keys)

    def random_deletion(self):
        """Delete random characters within a program."""
        print "Running random_deletion on {}".format(self.filename)
        program = self.program

        self.treemanager.import_file(program)
        assert self.parser.last_status == True

        self.text_compare(program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)
        random_lines = random_lines[:
                                    20]  # restrict to 20 lines to reduce runtime

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.treemanager.cursor_reset()
                self.log.append("self.treemanager.cursor_reset()")
                self.move(DOWN, linenr)
                self.log.append("self.move(DOWN, %s)" % linenr)
                self.move(RIGHT, col)
                self.log.append("self.move(RIGHT, %s)" % col)
                self.log.append("self.treemanager.key_delete()")
                x = self.treemanager.key_delete()
                if x == "eos":
                    continue
            self.treemanager.undo_snapshot()
            self.log.append("self.treemanager.undo_snapshot()")

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        parser.init_ast()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.set_font_test(7, 17)
        t1.import_file(self.program)

        assert self.parser.last_status == True
        assert parser.last_status == True

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)

    def random_insertion(self):
        """Insert random characters at random locations within a program."""
        print "Running random_insert on {}".format(self.filename)
        self.reset()

        self.treemanager.import_file(self.program)
        assert self.parser.last_status == True

        self.text_compare(self.program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.log.append("self.treemanager.cursor_reset()")
                self.log.append("self.move(DOWN, %s)" % linenr)
                self.log.append("self.move(RIGHT, %s)" % col)
                self.treemanager.cursor_reset()
                self.move(DOWN, linenr)
                self.move(RIGHT, col)
                k = self.get_random_key()
                self.log.append("self.treemanager.key_normal(%s)" % repr(k))
                x = self.treemanager.key_normal(k)
                if x == "eos":
                    continue
            self.log.append("self.treemanager.undo_snapshot()")
            self.treemanager.undo_snapshot()

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.import_file(self.program)

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)

    def random_insertdelete(self):
        """Insert and delete random characters at random locations within a
        program."""
        print "Running random_insertdelete on {}".format(self.filename)
        self.reset()

        self.treemanager.import_file(self.program)
        assert self.parser.last_status == True

        self.text_compare(self.program)

        line_count = len(self.treemanager.lines)
        random_lines = range(line_count)
        random.shuffle(random_lines)
        random_lines = random_lines[:
                                    20]  # restrict to 20 lines to reduce runtime

        start_version = self.treemanager.version
        for linenr in random_lines:
            cols = range(20)
            random.shuffle(cols)
            for col in cols:
                self.log.append("self.treemanager.cursor_reset()")
                self.log.append("self.move(%s, %s)" % (DOWN, linenr))
                self.log.append("self.move(%s, %s)" % (RIGHT, col))
                self.treemanager.cursor_reset()
                self.move(DOWN, linenr)
                self.move(RIGHT, col)
                k = self.get_random_key()
                if k in [
                        "a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7"
                ]:
                    # for a few characters DELETE instead of INSERT
                    self.log.append("self.treemanager.key_delete()")
                    x = self.treemanager.key_delete()
                else:
                    rk = self.get_random_key()
                    self.log.append("self.treemanager.key_normal(%s)" % rk)
                    x = self.treemanager.key_normal(rk)
                if x == "eos":
                    continue
            self.log.append("self.treemanager.undo_snapshot()")
            self.treemanager.undo_snapshot()

        end_version = self.treemanager.version
        broken = self.treemanager.export_as_text()

        # undo all and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        # redo all and compare with broken
        while self.treemanager.version < end_version:
            self.treemanager.key_shift_ctrl_z()
        self.text_compare(broken)

        # undo again and compare with original
        while self.treemanager.version > start_version:
            self.treemanager.key_ctrl_z()
        self.text_compare(self.program)

        t1 = TreeManager()
        parser, lexer = self.lang.load()
        t1.add_parser(parser, lexer, self.lang.name)
        t1.import_file(self.program)

        self.tree_compare(self.parser.previous_version.parent,
                          parser.previous_version.parent)

    def run(self):
        try:
            ft.random_deletion()
            self.reset()
            ft.random_insertion()
            self.reset()
            ft.random_insertdelete()
        except Exception as e:
            traceback.print_exc()
            print "Written log to 'fuzzy.log'."
            with open("fuzzy.log", "w") as f:
                for l in self.log:
                    f.write(l)
                    f.write("\n")
        else:
            print "Passed."
class FuzzyLboxStats:
    def __init__(self, main, sub):
        parser, lexer = main.load()
        self.lexer = lexer
        self.parser = parser
        self.ast = parser.previous_version
        self.treemanager = TreeManager()
        self.treemanager.add_parser(parser, lexer, main.name)
        self.treemanager.option_autolbox_insert = True
        self.langname = main.name

        parser.setup_autolbox(main.name, lexer)
        self.sub = sub

        self.inserted = 0
        self.log = []

    def load_main(self, filename):
        self.filename = filename
        f = open(filename, "r")
        self.content = f.read()
        f.close()
        self.content.replace("\n", "\r")
        self.treemanager.import_file(self.content)
        self.mainexprs = self.find_nonterms_by_name(self.treemanager,
                                                    self.main_repl_str)
        self.minver = self.treemanager.version

    def reset(self):
        self.parser.reset()
        self.ast = self.parser.previous_version
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, self.langname)
        self.treemanager.import_file(self.content)
        self.treemanager.option_autolbox_insert = True
        self.mainexprs = self.find_nonterms_by_name(self.treemanager,
                                                    self.main_repl_str)

    def load_expr(self, filename):
        f = open(filename, "r")
        content = f.read()
        f.close()
        self.replexprs = self.find_expressions(content, self.sub_repl_str)

    def load_expr_from_json(self, filename):
        import json
        with open(filename) as f:
            self.replexprs = json.load(f)

    def set_replace(self, main, sub):
        self.main_repl_str = main
        self.sub_repl_str = sub

    def find_nonterms_by_name(self, tm, name):
        l = []
        bos = tm.get_bos()
        eos = tm.get_eos()
        node = bos.right_sibling()
        while node is not eos:
            if validnonterm(node, name):
                l.append(node)
                node = next_node(node)
                continue
            if node.children:
                node = node.children[0]
            else:
                node = next_node(node)
        return l

    def find_expressions(self, program, expr):
        parser, lexer = self.sub.load()
        treemanager = TreeManager()
        treemanager.add_parser(parser, lexer, self.sub.name)
        treemanager.import_file(program)

        # find all sub expressions
        l = self.find_nonterms_by_name(treemanager, expr)
        return [subtree_to_text(st).rstrip() for st in l]

    def insert_python_expression(self, expr):
        for c in expr:
            self.treemanager.key_normal(c)

    def delete_expr(self, expr):
        # find first term and last term
        # select + delete
        node = expr
        while type(node.symbol) is Nonterminal:
            if node.children:
                node = node.children[0]
            else:
                node = next_node(node)
        first = node
        if isinstance(first, MultiTextNode):
            first = first.children[0]

        node = expr
        while type(node.symbol) is Nonterminal:
            if node.children:
                node = node.children[-1]
            else:
                node = prev_node(node)
        while node.lookup == "<ws>" or node.lookup == "<return>":
            node = node.prev_term
        last = node
        if isinstance(last, MultiTextNode):
            last = last.children[-1]

        if first.deleted or last.deleted:
            return None

        self.treemanager.select_nodes(first, last)
        deleted = self.treemanager.copySelection()
        self.treemanager.deleteSelection()
        return deleted

    def multi_len(self, autos):
        r = []
        for start, end, _, _ in autos:
            l = 0
            while start is not end:
                l += len(start.symbol.name)
                start = start.next_term
            r.append(l)
        return r

    def run(self, main_samples=None, sub_samples=None):
        assert len(self.treemanager.parsers) == 1

        ops = self.main_repl_str, len(
            [subtree_to_text(x) for x in self.mainexprs])
        preversion = self.treemanager.version

        inserted_error = 0
        inserted_valid = 0
        noinsert_error = 0
        noinsert_valid = 0
        noinsert_multi = 0

        # pick random exprs from main
        if not main_samples:
            samplesize = 10
            if len(self.mainexprs) < 10:
                samplesize = len(self.mainexprs)
            sample = random.sample(range(len(self.mainexprs)),
                                   samplesize)  # store this for repeatability
            exprchoices = [self.mainexprs[i] for i in sample]
            self.main_samples = sample
        else:
            self.main_samples = main_samples
            exprchoices = [self.mainexprs[i] for i in main_samples]

        if not sub_samples:
            # pick random exprs from sub
            sample = random.sample(range(len(self.replexprs)),
                                   len(exprchoices))
            replchoices = [self.replexprs[i] for i in sample]
            self.sub_samples = sample
        else:
            self.sub_samples = sub_samples
            replchoices = [self.replexprs[i] for i in sub_samples]

        for i in range(len(exprchoices)):
            e = exprchoices[i]
            if e.get_root() is None:
                continue
            before = len(self.treemanager.parsers)
            deleted = self.delete_expr(e)
            mboxes = None
            ilength = None
            if deleted:
                choice = replchoices[i].strip()
                choice_len = len(choice)
                if debug:
                    print "  Replacing '{}' with '{}':".format(
                        repr(truncate(deleted)), repr(choice))
                start = time.time()
                cursor = self.treemanager.cursor
                if cursor.node.lookup != "<ws>":
                    # Insert a leading space in situations like `return(x)`.
                    choice = " " + choice
                if cursor.node.symbol.name[
                        cursor.
                        pos:] == "" and cursor.node.next_term.lookup != "<ws>":
                    # Insert trailing space if there is none
                    choice = choice + " "
                self.insert_python_expression(choice)
                itime = time.time() - start
                valid = self.parser.last_status
                if before == len(self.treemanager.parsers):
                    if len(
                            self.parser.error_nodes
                    ) > 0 and self.parser.error_nodes[0].autobox and len(
                            self.parser.error_nodes[0].autobox) > 1:
                        noinsert_multi += 1
                        result = "No box inserted (Multi)"
                        outcome = "multi"
                        mboxes = self.multi_len(
                            self.parser.error_nodes[0].autobox)
                    elif valid:
                        noinsert_valid += 1
                        result = "No box inserted (Valid)"
                        outcome = "valid"
                    else:
                        noinsert_error += 1
                        result = "No box inserted (Error)"
                        outcome = "error"
                else:
                    result = "Box inserted"
                    innervalid = self.treemanager.parsers[-1][0].last_status
                    self.inserted += 1
                    recent_box = self.treemanager.parsers[-1][
                        0].previous_version.parent
                    lbox_len = self.lbox_length(recent_box)
                    ilength = (lbox_len, choice_len)
                    if valid and innervalid:
                        inserted_valid += 1
                        outcome = "ok"
                    else:
                        inserted_error += 1
                        outcome = "inerr"
                if debug: print "    => {} ({})".format(result, valid)
                nlboxes = len(self.treemanager.parsers)
                self.log.append([
                    self.filename,
                    repr(deleted),
                    repr(choice), outcome, itime, mboxes, ilength, nlboxes
                ])
            else:
                if debug:
                    print "Replacing '{}' with '{}':\n    => Already deleted".format(
                        truncate(subtree_to_text(e)), truncate(choice))
            self.undo(self.minver)
            exprchoices = [self.mainexprs[i] for i in self.main_samples]
        if debug:
            print("Boxes inserted: {}/{}".format(self.inserted, ops))
            print("Valid insertions:", inserted_valid)
            print("Invalid insertions:", inserted_error)
            print("No insertion (valid):", noinsert_valid)
            print("No insertion (error):", noinsert_error)
            print("No insertion (multi):", noinsert_multi)
        return (inserted_valid, inserted_error, noinsert_valid, noinsert_error,
                noinsert_multi)

    def undo(self, version):
        # reset everything
        self.reset()
        return
        while self.treemanager.version != version:
            before = self.treemanager.version
            self.treemanager.version -= 1
            self.treemanager.recover_version("undo",
                                             self.treemanager.version + 1)
            self.treemanager.cursor.load(self.treemanager.version,
                                         self.treemanager.lines)
            if before == self.treemanager.version:
                exit("Error")

    def lbox_length(self, root):
        l = []
        node = root.children[0]
        eos = root.children[-1]
        while node is not eos:
            if not node.deleted:
                l.append(node.symbol.name)
            node = node.next_term
        s = "".join(l).strip()
        return len(s)
예제 #17
0
class FuzzyMinimiser(object):

    def __init__(self, lang, program, test):
        grm = lang_dict[language]
        parser, lexer = grm.load()
        parser.init_ast()
        self.parser = parser
        self.lexer = lexer
        self.grm = grm
        self.treemanager = TreeManager()
        self.treemanager.add_parser(parser, lexer, grm.name)
        self.treemanager.version = 1
        self.treemanager.last_saved_version = 1

        self.program = program
        self.test = test
        self.pos = 0
        self.lastexception = None

    def reset(self):
        self.parser.reset()
        self.treemanager = TreeManager()
        self.treemanager.add_parser(self.parser, self.lexer, self.grm)

    def auto(self):
        logging.info("Running auto-mode")
        # try to reduce snapshots first
        while True:
            self.original = self.test[:] # remember original tests
            self.reduce("snapshot")
            if self.test == self.original:
                logging.info("No more snapshot reductions possible.")
                break
        # try single reduce next
        while True:
            self.original = self.test[:] # remember original tests
            self.reduce("single")
            if self.test == self.original:
                logging.info("No more single reductions possible.")
                break

    def reduce(self, mode):
        self.pos = 0
        logging.info("Pos: {} Len: {} Start {}-reduce".format(self.pos, len(self.test), mode))
        try:
            # Run without changes to store exception
            self.run()
        except Exception as e:
            self.lastexception = e
        while self.pos < len(self.test):
            self.delete(mode)
            try:
                self.run() # check if first run causes error
                logging.info("Pos: {} Len: {} Fixed".format(self.pos, len(self.test)))
                self.revert()
            except Exception as e:
                if not self.same_exception(e):
                    logging.info("Pos: {} Len: {} Wrong exception: '{}'".format(self.pos, len(self.test), e))
                    self.revert()
                    continue
                else:
                    logging.info("Pos: {} Len: {} Still failing: '{}'".format(self.pos, len(self.test), e))
                    self.save() # save progress
        self.save()

    def same_exception(self, e):
        if not self.lastexception:
            self.lastexception = e
            return True
        if type(self.lastexception) is type(e) and \
                self.lastexception.args == e.args:
                    return True
        return False

    def delete(self, mode):
        self.old = self.test[:] # copy
        if mode == "single":
            del self.test[self.pos:self.pos+4] # remove one keystroke
            if self.pos < len(self.test) and \
                self.test[self.pos] == "self.treemanager.undo_snapshot()":
                    self.test.pop(self.pos)
        elif mode == "snapshot":
            # removes 1 snapshot consisting of 20 keystrokes (each 4 lines) and
            # a `self.treemanager.undo_snapshot()` line at the end
            del self.test[self.pos:self.pos+81]

    def revert(self):
        diff = len(self.old) - len(self.test)
        self.test = self.old[:] # revert changes
        self.pos += diff # and skip test lines that are needed

    def set_destination(self, dest):
        filename = dest + ".1"
        i = 1
        while os.path.isfile(filename):
            filename = dest + ".{}".format(i)
            i += 1
        self.dest = filename
        logging.info("Setting destination to '{}'".format(filename))

    def save(self):
        f = open(self.dest, "w")
        for l in self.test:
            f.write(l)
            f.write("\n")
        f.close()

    def run(self):
        self.reset()
        self.treemanager.import_file(self.program)
        for l in self.test:
            exec(l)

    def move(self, direction, times):
        for i in range(times): self.treemanager.cursor_movement(direction)
예제 #18
0
class NodeEditor(QFrame):

    # ========================== init stuff ========================== #

    def __init__(self, parent=None):
        QtGui.QWidget.__init__(self, parent)

        self.viewport_y = 0 # top visible line
        self.imagemode = False
        self.image = None

        self.scroll_height = 0
        self.scroll_width = 0

        self.timer = QTimer(self)
        self.backuptimer = QTimer(self)
        self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer)
        self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer)
        self.backuptimer.start(30000)
        self.undotimer = QTimer(self)
        self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer)

        self.blinktimer = QTimer(self)
        self.blinktimer.start(500)
        self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer)
        self.show_cursor = True

        self.boxcolors = [QColor("#DC322F"), QColor("#268BD2"), QColor("#2Ac598"), QColor("#D33682"), QColor("#B58900"), QColor("#859900")]
        self.setCursor(Qt.IBeamCursor)

        # Semi-transparent overlay.
        # Used to display heat-map visualisation of profiler info, etc.
        self.overlay = Overlay(self)
        # Always show the overlay, users hide visualisations with the HUD.
        self.overlay.show()

        # Show / don't show HUD visualisations.
        self.hud_callgraph = False
        self.hud_eval = False
        self.hud_heat_map = False
        self.hud_types = False

    def hud_show_callgraph(self):
        self.hud_callgraph = True
        self.hud_eval = False
        self.hud_heat_map = False
        self.hud_types = False

    def hud_show_eval(self):
        self.hud_callgraph = False
        self.hud_eval = True
        self.hud_heat_map = False
        self.hud_types = False

    def hud_show_types(self):
        self.hud_callgraph = False
        self.hud_eval = False
        self.hud_heat_map = False
        self.hud_types = True

    def hud_show_heat_map(self):
        self.hud_callgraph = False
        self.hud_eval = False
        self.hud_heat_map = True
        self.hud_types = False

    def hud_off(self):
        self.hud_callgraph = False
        self.hud_eval = False
        self.hud_heat_map = False
        self.hud_types = False

    def focusOutEvent(self, event):
        self.blinktimer.stop()
        self.show_cursor = True
        self.update()

    def focusInEvent(self, event):
        self.blinktimer.start()

    def resizeEvent(self, event):
        self.overlay.resize(event.size())
        event.accept()

    def analysis_timer(self):
        if self.getWindow().show_namebinding():
            self.tm.analyse()
            self.update()
        self.timer.stop()

        # save swap
        filename = self.getEditorTab().filename
        if filename:
            self.saveToJson(filename + ".swp", True)

    def backup_timer(self):
        filename = self.getEditorTab().filename
        if filename:
            self.saveToJson(filename + ".bak", True)

    def trigger_blinktimer(self):
        if self.timer.isActive():
            self.show_cursor = True
            return
        self.show_cursor ^= True
        self.update()

    def trigger_undotimer(self):
        self.tm.undo_snapshot()
        self.undotimer.stop()

    def setImageMode(self, boolean):
        self.imagemode = boolean

    def reset(self):
        self.update()

    def set_mainlanguage(self, parser, lexer, lang_name):
        self.tm = TreeManager()
        self.tm.add_parser(parser, lexer, lang_name)

    def get_mainlanguage(self):
        return self.tm.parsers[0][2]

    def set_sublanguage(self, language):
        self.sublanguage = language

    def event(self, event):
        if event.type() == QEvent.ToolTip:
            pos = event.pos()
            temp_cursor = self.tm.cursor.copy()
            result = self.coordinate_to_cursor(pos.x(), pos.y())
            node = self.tm.cursor.node
            self.tm.cursor.line = temp_cursor.line
            self.tm.cursor.node = temp_cursor.node
            self.tm.cursor.pos = temp_cursor.pos
            if not result:
                QToolTip.hideText()
                event.ignore()
                return True
            # Draw errors, if there are any.
            msg = self.tm.get_error(node)
            if msg:
                QToolTip.showText(event.globalPos(), msg)
                return True
            # Draw tooltips if there are any. Tooltips can appear with any HUD
            # visualisation.
            elif (self.hud_callgraph or self.hud_eval or self.hud_types or
                  self.hud_heat_map):
                strings = []
                annotes = node.get_annotations_with_hint(ToolTip)
                if self.hud_callgraph:
                    for annote in annotes:
                        if annote.has_hint(HUDCallgraph):
                            strings.append(annote.annotation)
                elif self.hud_eval:
                    for annote in annotes:
                        if annote.has_hint(HUDEval):
                            strings.append(annote.annotation)
                elif self.hud_types:
                    for annote in annotes:
                        if annote.has_hint(HUDTypes):
                            strings.append(annote.annotation)
                elif self.hud_heat_map:
                    for annote in annotes:
                        if annote.has_hint(HUDHeatmap):
                            strings.append(annote.annotation)
                msg = "\n".join(strings)
                if msg.strip() != "":
                    if self.tm.tool_data_is_dirty:
                        msg += "\n[Warning: Information may be out of date.]"
                    QToolTip.showText(event.globalPos(), msg)
                    return True
            QToolTip.hideText()
            event.ignore()
            return True
        return QFrame.event(self, event)

    # ========================== GUI related stuff ========================== #

    def blink(self):
        if self.show_cursor:
            self.show_cursor = 0
        else:
            self.show_cursor = 1
        self.update()

    def sliderChanged(self, value):
        self.viewport_y = value
        self.overlay.start_line = self.viewport_y
        self.update()

    def sliderXChanged(self, value):
        self.move(-value*self.fontwt,0)
        self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height())
        if self.x() == 0:
            self.updateGeometry()
        self.update()

    def getScrollSizes(self):
        total_lines = 0
        max_width = 0
        for l in self.lines:
            total_lines += l.height
            max_width = max(max_width, l.width)
        max_visible_lines = self.geometry().height() / self.fontht
        self.scroll_height = max(0, total_lines - max_visible_lines)

        current_width = self.parentWidget().geometry().width() / self.fontwt
        self.scroll_width = max(0, max_width - current_width)

    def paintEvent(self, event):
        # Clear data in the visualisation overlay
        self.overlay.clear_data()

        gfont = QApplication.instance().gfont
        self.font = gfont.font
        self.fontwt = gfont.fontwt
        self.fontht = gfont.fontht
        self.fontd  = gfont.fontd
        QtGui.QFrame.paintEvent(self, event)
        paint = QtGui.QPainter()
        if self.imagemode:
            self.image = QImage()
            paint.begin(self.image)
        else:
            paint.begin(self)
        paint.setFont(self.font)

        self.longest_column = 0

        # calculate how many lines we need to show
        self.init_height = self.geometry().height()

        self.paintLines(paint, self.viewport_y)

        paint.end()

        total_lines = 0
        max_width = 0
        for l in self.lines:
            total_lines += l.height
            max_width = max(max_width, l.width)
        max_visible_lines = self.geometry().height() / self.fontht
        self.scroll_height = max(0, total_lines - max_visible_lines)

        current_width = self.parentWidget().geometry().width() / self.fontwt
        self.scroll_width = max(0, max_width - current_width)

        railroad_annotations = self.tm.get_all_annotations_with_hint(Railroad)
        self.overlay.add_railroad_data(railroad_annotations)

        self.emit(SIGNAL("painted()"))

    # paint lines using new line manager
    def paintLines(self, paint, startline):

        # find internal line corresponding to visual line
        visual_line = 0
        internal_line = 0
        for l in self.tm.lines:
            if visual_line + l.height > startline:
                break
            visual_line += l.height
            internal_line += 1

        x = 0
        y = visual_line - startline # start drawing outside of viewport to display partial images
        self.paint_start = (internal_line, y)

        max_y = self.geometry().height() / self.fontht

        line = internal_line
        node = self.tm.lines[line].node

        _, _, self.end_line = self.paint_nodes(paint, node, x, y, line, max_y)


    #XXX if starting node is inside language box, init lbox with amount of language boxes
    def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0):

        settings = QSettings("softdev", "Eco")
        colors = self.boxcolors
        if settings.value("app_theme", "Light").toString() in ["Dark"]:
            alpha = 100
            self.highlight_line_color = QColor(250,250,250,20)
        elif settings.value("app_theme", "Light").toString() in ["Gruvbox"]:
            alpha = 100
            self.highlight_line_color = QColor(250,250,250,20)
        else:
            alpha = 60
            self.highlight_line_color = QColor(0,0,0,10)
        self.show_highlight_line = settings.value("highlight_line", False).toBool()

        first_node = node
        selected_language = self.tm.mainroot
        error_node = self.tm.get_mainparser().error_node
        error_node = self.fix_errornode(error_node)

        highlighter = self.get_highlighter(node)
        selection_start = min(self.tm.selection_start, self.tm.selection_end)
        selection_end = max(self.tm.selection_start, self.tm.selection_end)
        draw_selection_start = (0,0,0)
        draw_selection_end = (0,0,0)
        start_lbox = self.get_languagebox(node)
        renderer = self.get_renderer(node)

        self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node)
        #XXX get initial x for langbox

        if start_lbox:
            lbox += 1
        if start_lbox and self.selected_lbox is start_lbox:
            draw_lbox = True
        else:
            draw_lbox = False

        draw_all_boxes = self.getWindow().show_languageboxes()

        self.lines = self.tm.lines
        self.cursor = self.tm.cursor
        self.lines[line].height = 1 # reset height
        draw_cursor = True
        show_namebinding = self.getWindow().show_namebinding()
        while y < max_y:

            # if we found a language box, continue drawing inside of it
            if isinstance(node.symbol, MagicTerminal):
                lbox += 1
                lbnode = node.symbol.ast
                if self.selected_lbox is node:
                    color = colors[(lbox-1) % len(colors)]
                    self.draw_lbox_bracket(paint, '[', node, x, y, color)
                    draw_lbox = True
                    selected_language = lbnode
                else:
                    draw_lbox = False
                node = lbnode.children[0]
                highlighter = self.get_highlighter(node)
                renderer = self.get_renderer(node)
                error_node = self.tm.get_parser(lbnode).error_node
                error_node = self.fix_errornode(error_node)
                continue

            if isinstance(node, EOS):
                lbnode = self.get_languagebox(node)
                if self.cursor.node is lbnode:
                    self.draw_cursor(paint, x, 4 + y * self.fontht)
                if lbnode:
                    color = colors[(lbox-1) % len(colors)]
                    if lbox > 0:
                        lbox -= 1
                    node = lbnode.next_term
                    highlighter = self.get_highlighter(node)
                    renderer = self.get_renderer(node)
                    if self.selected_lbox is lbnode:
                        # draw bracket
                        self.draw_lbox_bracket(paint, ']', node, x, y, color)
                        draw_lbox = False
                    lbnode = self.get_languagebox(node)
                    if lbnode and self.selected_lbox is lbnode:
                        draw_lbox = True
                        error_node = self.tm.get_parser(lbnode.symbol.ast).error_node
                        error_node = self.fix_errornode(error_node)
                    else:
                        error_node = self.tm.get_mainparser().error_node
                        error_node = self.fix_errornode(error_node)
                    continue
                else:
                    self.lines[line].width = x / self.fontwt
                    break

            # draw language boxes
            if lbox > 0 and (draw_lbox or draw_all_boxes):
                if draw_all_boxes:
                    color = colors[(lbox-1) % len(colors)]
                    color.setAlpha(alpha)
                else:
                    color = colors[0]
                    color.setAlpha(alpha)
                if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box
                    color.setAlpha(20)
                renderer.update_image(node)
                if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal):
                    if not node.image or node.plain_mode:
                        if isinstance(node, BOS) and isinstance(node.next_term, EOS):
                            self.draw_lbox_hints(paint, node, x, y, color)
                        else:
                            paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color)

            # prepare selection drawing
            if node is selection_start.node:
                if node.lookup == "<return>":
                    sel_x = x
                else:
                    sel_x = x + selection_start.pos * self.fontwt
                draw_selection_start = (sel_x, y, line)

            if node is selection_end.node:
                draw_selection_end = (x + selection_end.pos * self.fontwt, y, line)


            # draw node
            dx, dy = renderer.paint_node(paint, node, x, y, highlighter)
            x += dx
            self.lines[line].height = max(self.lines[line].height, dy)

            # Draw footnotes and add heatmap data to overlay.
            annotes = [annote.annotation for annote in node.get_annotations_with_hint(Heatmap)]
            # Heatmap data can always be sent to the overlay, it won't be
            # rendered unless the user selects the Heatmap HUD radio button.
            for annote in annotes:
                self.overlay.add_heatmap_datum(line + 1, annote)
            if self.hud_eval or self.hud_types:
                infofont = QApplication.instance().tool_info_font
                annotes = []
                annotes_ = node.get_annotations_with_hint(Footnote)
                if self.hud_eval:
                    for annote in annotes_:
                        if annote.has_hint(HUDEval):
                            annotes.append(annote)
                elif self.hud_types:
                    for annote in annotes_:
                        if annote.has_hint(HUDTypes):
                            annotes.append(annote)
                footnote = " ".join([annote.annotation for annote in annotes])
                if footnote.strip() != "":
                    if not self.tm.tool_data_is_dirty:
                        infofont.font.setBold(True)
                    else:
                        infofont.font.setBold(False)
                    paint.setFont(infofont.font)
                    paint.setPen(QPen(QColor((highlighter.get_default_color()))))
                    start_y = self.fontht + ((y + 1) * self.fontht)
                    paint.drawText(QtCore.QPointF(x-dx, start_y), footnote)
                    self.lines[line].height = max(self.lines[line].height, 2)
                    paint.setFont(self.font)

            # after we drew a return, update line information
            if node.lookup == "<return>" and not node is first_node:
                # draw lbox to end of line
                if draw_lbox or (draw_all_boxes and lbox > 0):
                    paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color)

                self.lines[line].width = x / self.fontwt
                x = 0
                y += self.lines[line].height
                line += 1
                self.lines[line].height = 1 # reset height

            if self.show_highlight_line:
                if node.lookup == "<return>" or isinstance(node, BOS):
                    if self.cursor.line == line:
                        self.highlight_line(paint, y)

            # draw cursor
            if node is self.cursor.node and self.show_cursor:
                draw_x = max(0, x-dx)
                cursor_pos = self.cursor.pos

                if node.symbol.name == "\r":
                    cursor_pos = 0
                if node.image and not node.plain_mode:
                    draw_x = x
                    cursor_pos = 0
                self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 4 + y * self.fontht)


            if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor:
                draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3)
                paint.drawRect(draw_cursor_at)

                # set lbox info coordinates
                infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht)
                draw_cursor = False

            # draw squiggly line
            if node is error_node or (show_namebinding and self.tm.has_error(node)):
                if isinstance(node, EOS):
                    length = self.fontwt
                else:
                    length = len(node.symbol.name)*self.fontwt
                if isinstance(node.symbol, MagicTerminal):
                    self.draw_vertical_squiggly_line(paint,x,y)
                else:
                    if self.tm.has_error(node):
                        err_color = "orange"
                    else:
                        err_color = "red"
                    self.draw_squiggly_line(paint,x-length,y,length, err_color)

            node = node.next_term

        if selection_start != selection_end:
            self.draw_selection(paint, draw_selection_start, draw_selection_end, max_y)

        # paint infobox
        if False:
            lang_name = self.tm.parsers[selected_language]
            lang_status = self.tm.get_parser[selected_language][0].last_status
            if lang_status is True:
                color = QColor(100,255,100)
            else:
                color = QColor(255,100,100)
            paint.setFont(infofont)
            paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*infofontwt, infofontht), color)
            paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name)
            paint.setFont(self.font)

        return x, y, line

    def highlight_line(self, paint, y):
        width = self.parentWidget().geometry().width()
        paint.fillRect(QRect(-5, y * self.fontht + 3, width, self.fontht), self.highlight_line_color)

    def draw_lbox_hints(self, paint, node, x, y, color):
        if node is self.cursor.node:
            return
        alpha = color.alpha()
        color.setAlpha(255)
        path = QPainterPath()
        x = x - 2
        y = y*self.fontht + 4

        path.moveTo(x, y)
        path.lineTo(x+6, y)
        path.lineTo(x+3, y+3)
        path.lineTo(x, y)

        paint.fillPath (path, QBrush (color));
        color.setAlpha(alpha)

    def draw_lbox_bracket(self, paint, bracket, node, x, y, color):
        assert bracket in ['[',']']
        oldpen = paint.pen()
        newpen = QPen()
        color.setAlpha(255)
        newpen.setColor(color)
        newpen.setWidth(1)
        paint.setPen(newpen)

        # paint brackets
        path = QPainterPath()
        if bracket == '[':
            if x == 0:
                tmpx = x + 2
            else:
                tmpx = x + 1 # adjust bracket position
            path.moveTo(tmpx,   3+y*self.fontht)
            path.lineTo(tmpx-2, 3+y*self.fontht)
            path.moveTo(tmpx-2, 3+y*self.fontht)
            path.lineTo(tmpx-2, 3+y*self.fontht + self.fontht - 1)
        else:
            tmpx = x - 1
            path.moveTo(tmpx,   3+y*self.fontht)
            path.lineTo(tmpx+2, 3+y*self.fontht)
            path.moveTo(tmpx+2, 3+y*self.fontht)
            path.lineTo(tmpx+2, 3+y*self.fontht + self.fontht - 1)
        path.lineTo(tmpx, 3+y*self.fontht + self.fontht - 1)
        paint.drawPath(path)

        paint.setPen(oldpen)

    def fix_errornode(self, error_node):
        if not error_node:
            return
        while isinstance(error_node.symbol, IndentationTerminal):
            error_node = error_node.next_term
        return error_node

    def draw_cursor(self, paint, x, y):
        pen = paint.pen()
        colorhex = self.palette().color(QPalette.Text)
        pen.setColor(QColor(colorhex))
        paint.setPen(pen)
        draw_cursor_at = QRect(x, y, 0, self.fontht - 3)
        paint.drawRect(draw_cursor_at)

    def draw_vertical_squiggly_line(self, paint, x, y):
        paint.setPen(Qt.CustomDashLine)
        pen = paint.pen()
        pen.setDashPattern([2,2])
        pen.setColor(QColor("red"))
        paint.setPen(pen)
        y = 3+y*self.fontht
        paint.drawLine(x-1, y, x-1, y+self.fontht)
        paint.drawLine(x, y+2, x, y+self.fontht)
        paint.setPen(Qt.SolidLine)

    def draw_squiggly_line(self, paint, x, y, length, color):
        paint.setPen(Qt.CustomDashLine)
        pen = paint.pen()
        pen.setDashPattern([2,2])
        pen.setColor(QColor(color))
        paint.setPen(pen)
        y = (y+1)*self.fontht+1
        paint.drawLine(x, y, x+length, y)
        paint.drawLine(x+2, y+1, x+2+length, y+1)
        paint.setPen(Qt.SolidLine)

    def draw_selection(self, paint, draw_selection_start, draw_selection_end, max_y):
        x1, y1, line1 = draw_selection_start
        x2, y2, line2 = draw_selection_end
        start = min(self.tm.selection_start, self.tm.selection_end)
        end = max(self.tm.selection_start, self.tm.selection_end)
        if x1 + y1 + line1 + x2 + y2 + line2 == 0:
            # everything out of viewport, draw nothing
            # unless start and end are on opposite sides of the viewport
            if not(start.line <= self.paint_start[0] and end.line >= self.paint_start[0] + max_y):
                    return
        if x1 + y1 + line1 == 0:
            # start outside of viewport
            line1 = self.paint_start[0]
        if x2 + y2 + line2 == 0:
            # end outside of viewport
            line2 = self.paint_start[0] + max_y
            y2 = max_y
        if y1 == y2:
            paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100))
        else:
            width = max(self.fontwt, self.tm.lines[line1].width*self.fontwt)
            paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, width - x1, self.fontht), QColor(0,0,255,100))
            y = y1 + self.tm.lines[line1].height
            for i in range(line1+1, line2):
                width = self.tm.lines[i].width*self.fontwt
                if width == 0:
                    width = self.fontwt
                paint.fillRect(QRectF(0, 3 + y * self.fontht, width, self.fontht), QColor(0,0,255,100))
                y = y + self.tm.lines[i].height
            paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100))

    def get_highlighter(self, node):
        root = node.get_root()
        base = lang_dict[self.tm.get_language(root)].base
        s = syntaxhighlighter.get_highlighter(base, self.palette())
        return s

    def get_languagebox(self, node):
        root = node.get_root()
        lbox = root.get_magicterminal()
        return lbox

    def get_renderer(self, node):
        root = node.get_root()
        base = lang_dict[self.tm.get_language(root)].base
        return renderers.get_renderer(base, self.fontwt, self.fontht, self.fontd)

    def focusNextPrevChild(self, b):
        # don't switch to next widget on TAB
        return False

    def mousePressEvent(self, e):
        if e.button() == Qt.LeftButton:
            self.tm.input_log.append("# mousePressEvent")
            self.coordinate_to_cursor(e.x(), e.y())
            self.tm.selection_start = self.tm.cursor.copy()
            self.tm.selection_end = self.tm.cursor.copy()
            self.tm.input_log.append("self.selection_start = self.cursor.copy()")
            self.tm.input_log.append("self.selection_end = self.cursor.copy()")
            self.getWindow().showLookahead()
            self.update()

    def mouseDoubleClickEvent(self, e):
        if e.button() == Qt.LeftButton:
            self.coordinate_to_cursor(e.x(), e.y())
            node = self.tm.get_node_from_cursor()
            lbox = self.get_languagebox(node)
            if lbox and lbox.symbol.name == "<IPython>":
                if lbox.plain_mode is False:
                    lbox.plain_mode = True
                else:
                    lbox.plain_mode = False
                self.update()
                return
            elif node.image is None:
                self.tm.doubleclick_select()
                self.update()
                return

            if node.plain_mode is False:
                node.plain_mode = True
                self.tm.cursor.pos = len(node.symbol.name)
            else:
                node.plain_mode = False
            self.update()

    def cursor_to_coordinate(self):
        y = 0
        for l in self.tm.lines[:self.cursor.line]:
            y += l.height * self.fontht
        x = self.tm.cursor.get_x() * self.fontwt
        y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht
        return (x,y)

    def coordinate_to_cursor(self, x, y):
        mouse_y = y / self.fontht
        first_line = self.paint_start[0]
        y_offset = self.paint_start[1]

        y = y_offset
        line = first_line
        if mouse_y < 0:
            while line > 0:
                y -= self.tm.lines[line].height
                if y < mouse_y:
                    break
                line -= 1
        else:
            while line < len(self.tm.lines) - 1:
                y += self.tm.lines[line].height
                if y > mouse_y:
                    break
                line += 1

        self.tm.cursor.line = line
        cursor_x = int(round(float(x) / self.fontwt))
        self.tm.cursor.move_to_x(cursor_x)

        self.tm.input_log.append("self.cursor.line = %s" % str(line))
        self.tm.log_input("cursor.move_to_x", str(cursor_x))

        if mouse_y > y or self.tm.cursor.get_x() != cursor_x:
            return False
        return True

    def mouseMoveEvent(self, e):
        # apparently this is only called when a mouse button is clicked while
        # the mouse is moving
        if self.tm.input_log[-2].startswith("self.cursor.move_to_x"):
            # only log the last move event
            self.tm.input_log.pop()
            self.tm.input_log.pop()
            self.tm.input_log.pop()
        self.coordinate_to_cursor(e.x(), e.y())
        self.tm.selection_end = self.tm.cursor.copy()
        self.tm.input_log.append("self.selection_end = self.cursor.copy()")
        self.update()
        self.getEditorTab().keypress()

    def keyPressEvent(self, e):
        self.timer.start(500)
        self.show_cursor = True

        startundotimer = False
        key = KeyPress(e)

        # key presses to ignore
        if key.is_modifier or key.page_up or key.page_down:
            return

        # has been processes in get_nodes_at_pos -> reset
        self.edit_rightnode = False

        if key.escape:
            self.tm.key_escape()
        elif key.backspace:
            startundotimer = True
            self.tm.key_backspace()
        elif key.home:
            self.tm.key_home(shift=key.m_shift)
        elif key.end:
            self.tm.key_end(shift=key.m_shift)
        elif key.is_arrow:
            if key.jump_word:
                self.tm.ctrl_cursor(key, shift=key.m_shift)
            else:
                self.tm.key_cursors(key, shift=key.m_shift)
        elif key.delete:
            startundotimer = True
            self.tm.key_delete()
        elif e.key() == Qt.Key_F3:
            self.tm.find_next()
        # User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't
        # sensibly insert into the text.
        elif key.has_action_modifier:
            pass
        # every other normal key press
        else:
            startundotimer = True
            if e.key() == Qt.Key_Tab:
                text = "    "
            else:
                # text is a char array, so we need the first letter
                # to match it against the set
                text = e.text()
                if text.isEmpty() or text.toUtf8()[0] not in whitelist:
                    logging.debug("Key %s not supported" % text)
                    return
            self.tm.key_normal(text)

        self.getWindow().btReparse([])
        self.update()
        self.emit(SIGNAL("keypress(QKeyEvent)"), e)
        self.getWindow().showLookahead()

        if startundotimer:
            self.undotimer.start(500)

    def showLanguageBoxMenu(self):
        self.showSubgrammarMenu()
        self.create_languagebox()

    def create_languagebox(self):
        if self.sublanguage:
            if self.tm.hasSelection():
                self.tm.surround_with_languagebox(self.sublanguage)
            else:
                self.tm.add_languagebox(self.sublanguage)

    def change_languagebox(self):
        if self.sublanguage:
            self.tm.change_languagebox(self.sublanguage)

    def showCodeCompletion(self):
        l = self.tm.getCompletion()
        if l:
            self.showCodeCompletionMenu(l)

    def println(self, prestring, y):
        node = self.lines[y].node.next_term
        x = []
        while node is not None and node.symbol.name != "\r":
            x.append(node.symbol.name)
            node = node.next_term
        print(prestring, "".join(x))

    def print_line(self, y):
        current = self.lines[y].node
        while True:
            print(current)
            current = current.next_term
            if current is None:
                return

    # ========================== AST modification stuff ========================== #

    def insertTextNoSim(self, text):
        self.viewport_y = 0
        self.tm.import_file(text)
        return

    def getTL(self):
        return self.getWindow().tl

    def getPL(self):
        return self.getWindow().pl

    def getLRP(self):
        return self.getWindow().lrp

    def getWindow(self):
        return self.window()

    def getEditorTab(self):
        return self.parent().parent().parent()

    def getScrollArea(self):
        return self.parent().parent()

    def createSubgrammarMenu(self, menu, change=False):
        self.sublanguage = None

        tmp = None
        if change:
            # try and find lbox and set cursor to previous node before getting
            # lookahead list
            root = self.tm.cursor.node.get_root()
            lbox = root.get_magicterminal()
            if lbox:
                tmp = self.tm.cursor.node
                self.tm.cursor.node = lbox.prev_term

        # Create actions
        bf = QFont()
        bf.setBold(True)
        valid_langs = []
        for l in languages:
            if "<%s>" % l in self.tm.getLookaheadList():
                valid_langs.append(l)

        if tmp:
            # undo cursor change
            self.tm.cursor.node = tmp
        if len(valid_langs) > 0:
            for l in valid_langs:
                item = QAction(str(l), menu)
                item.setData(l)
                self._set_icon(item, l)
                item.setFont(bf)
                menu.addAction(item)
            menu.addSeparator()
        for l in languages:
            item = QAction(str(l), menu)
            item.setData(l)
            self._set_icon(item, l)
            menu.addAction(item)
        return menu

    def showSubgrammarMenu(self):
        menu = QtGui.QMenu("Language", self)
        self.createSubgrammarMenu(menu)
        x,y = self.cursor_to_coordinate()
        action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
        if action:
            self.sublanguage = action.data().toPyObject()
            self.edit_rightnode = True

    def _set_icon(self, mitem, lang):
        if lang.base.lower() == "html":
            icon = QIcon.fromTheme("text-xhtml+xml")
        else:
            icon = QIcon.fromTheme("text-x-" + lang.base.lower())
        if icon.isNull():
            icon = QIcon.fromTheme("application-x-" + lang.base.lower())
            if icon.isNull():
                icon = QIcon.fromTheme("text-x-generic")
        mitem.setIcon(icon)

    def showCodeCompletionMenu(self, l):
        menu = QtGui.QMenu( self )
        # Create actions
        toolbar = QtGui.QToolBar()
        for n in l:
            path = []
            for p in n.path:
                if p and p.name:
                    path.append(p.name)
            if n.vartype:
                vartype = n.vartype
                while vartype.children != []:
                    try:
                        vartype = vartype.children[0]
                    except KeyError:
                        vartype = vartype.children["name"]
                text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind)
            elif n.kind == "method":
                text = self.cc_method(n) + " - %s" % (".".join(path))
            else:
                text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind)
            item = toolbar.addAction(text, self.createCCFunc(n.name))
            item.setIcon(QIcon("gui/" + n.kind + ".png"))
            menu.addAction(item)
        x,y = self.cursor_to_coordinate()
        menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))

    def cc_method(self, n):
        s = [n.name, "("]
        param_ln = n.astnode.children["params"]
        if isinstance(param_ln, ListNode):
            for p in param_ln.children:
                tmp = p.children["type"]
                if isinstance(tmp, AstNode):
                    s.append(tmp.children["name"].symbol.name)
                else:
                    s.append(tmp.symbol.name)
                s.append(" ")
                s.append(p.children["name"].symbol.name)
                if p != param_ln.children[-1]:
                    s.append(", ")
        s.append(")")
        return "".join(s)

    def createMenuFunction(self, l):
        def action():
            self.sublanguage = l
            self.edit_rightnode = True
        return action

    def createCCFunc(self, text):
        def action():
            self.tm.pasteCompletion(text)
        return action

    def selectSubgrammar(self, item):
        pass

    def saveToJson(self, filename, swap=False):
        whitespaces = self.tm.get_mainparser().whitespaces
        root = self.tm.parsers[0][0].previous_version.parent
        language = self.tm.parsers[0][2]
        manager = JsonManager()
        manager.save(root, language, whitespaces, filename)
        if not swap:
            self.tm.changed = False
            self.emit(SIGNAL("painted()"))

    def loadFromJson(self, filename):
        manager = JsonManager()
        language_boxes = manager.load(filename)

        self.tm = TreeManager()

        self.tm.load_file(language_boxes)
        self.reset()

    def export(self, run=False, profile=False, source=None, debug=False):
        return self.tm.export(None, run, profile, source=source, debug=debug)