def random_insertion(self): """Insert random characters at random locations within a program.""" print "Running random_insert on {}".format(self.filename) self.reset() self.treemanager.import_file(self.program) assert self.parser.last_status == True self.text_compare(self.program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.log.append("self.treemanager.cursor_reset()") self.log.append("self.move(DOWN, %s)" % linenr) self.log.append("self.move(RIGHT, %s)" % col) self.treemanager.cursor_reset() self.move(DOWN, linenr) self.move(RIGHT, col) k = self.get_random_key() self.log.append("self.treemanager.key_normal(%s)" % repr(k)) x = self.treemanager.key_normal(k) if x == "eos": continue self.log.append("self.treemanager.undo_snapshot()") self.treemanager.undo_snapshot() end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) t1 = TreeManager() parser, lexer = self.lang.load() t1.add_parser(parser, lexer, self.lang.name) t1.import_file(self.program) self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent)
def find_expressions(self, program, expr): parser, lexer = self.sub.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, self.sub.name) treemanager.import_file(program) # find all sub expressions l = self.find_nonterms_by_name(treemanager, expr) return [subtree_to_text(st).rstrip() for st in l]
def find_subtree(grammar, program, cond): parser, lexer = grammar.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, grammar.name) try: treemanager.import_file(program) except Exception: return None if parser.last_status is False: return None # find all sub expressions l = find_nonterms_by_name(treemanager, cond) return [subtree_to_text(st).rstrip() for st in l]
def run(filename): # Load grammar parser, lexer = java.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, "") # Load file print("Running", filename) with open(filename, "r") as f: treemanager.import_file(f.read()) if not parser.last_status: print("Couldn't parse file. Skip.") return filesize = os.path.getsize(filename) timings = [] # Insert `1+` after every `=` node = treemanager.get_bos() while type(node) is not EOS: if node.symbol.name == "=": node.insert("1+", len(node.symbol.name)) node.mark_changed() treemanager.relex(node) treemanager.post_keypress("") treemanager.save_current_version() parser.prev_version = treemanager.version parser.reference_version = treemanager.reference_version start = time() parser.inc_parse() end = time() parser.top_down_reuse() treemanager.save_current_version( postparse=True) # save post parse tree if parser.last_status: treemanager.reference_version = treemanager.version TreeManager.version = treemanager.version if not parser.last_status: break timings.append(str(end - start)) node = node.next_term if len(timings) > 0: with open("results.csv", "a+") as f: f.write("{} {} {}".format(filename, filesize, ",".join(timings))) f.write("\n")
class FuzzyMinimiser(object): def __init__(self, lang, program, test): grm = lang_dict[language] parser, lexer = grm.load() parser.init_ast() self.parser = parser self.lexer = lexer self.grm = grm self.treemanager = TreeManager() self.treemanager.add_parser(parser, lexer, grm.name) self.treemanager.version = 1 self.treemanager.last_saved_version = 1 self.program = program self.test = test self.pos = 0 self.lastexception = None def reset(self): self.parser.reset() self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, self.grm) def auto(self): logging.info("Running auto-mode") # try to reduce snapshots first while True: self.original = self.test[:] # remember original tests self.reduce("snapshot") if self.test == self.original: logging.info("No more snapshot reductions possible.") break # try single reduce next while True: self.original = self.test[:] # remember original tests self.reduce("single") if self.test == self.original: logging.info("No more single reductions possible.") break def reduce(self, mode): self.pos = 0 logging.info("Pos: {} Len: {} Start {}-reduce".format(self.pos, len(self.test), mode)) try: # Run without changes to store exception self.run() except Exception as e: self.lastexception = e while self.pos < len(self.test): self.delete(mode) try: self.run() # check if first run causes error logging.info("Pos: {} Len: {} Fixed".format(self.pos, len(self.test))) self.revert() except Exception as e: if not self.same_exception(e): logging.info("Pos: {} Len: {} Wrong exception: '{}'".format(self.pos, len(self.test), e)) self.revert() continue else: logging.info("Pos: {} Len: {} Still failing: '{}'".format(self.pos, len(self.test), e)) self.save() # save progress self.save() def same_exception(self, e): if not self.lastexception: self.lastexception = e return True if type(self.lastexception) is type(e) and \ self.lastexception.args == e.args: return True return False def delete(self, mode): self.old = self.test[:] # copy if mode == "single": del self.test[self.pos:self.pos+4] # remove one keystroke if self.pos < len(self.test) and \ self.test[self.pos] == "self.treemanager.undo_snapshot()": self.test.pop(self.pos) elif mode == "snapshot": # removes 1 snapshot consisting of 20 keystrokes (each 4 lines) and # a `self.treemanager.undo_snapshot()` line at the end del self.test[self.pos:self.pos+81] def revert(self): diff = len(self.old) - len(self.test) self.test = self.old[:] # revert changes self.pos += diff # and skip test lines that are needed def set_destination(self, dest): filename = dest + ".1" i = 1 while os.path.isfile(filename): filename = dest + ".{}".format(i) i += 1 self.dest = filename logging.info("Setting destination to '{}'".format(filename)) def save(self): f = open(self.dest, "w") for l in self.test: f.write(l) f.write("\n") f.close() def run(self): self.reset() self.treemanager.import_file(self.program) for l in self.test: exec(l) def move(self, direction, times): for i in range(times): self.treemanager.cursor_movement(direction)
class FuzzyLboxStats: def __init__(self, main, sub): parser, lexer = main.load() self.lexer = lexer self.parser = parser self.ast = parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(parser, lexer, main.name) self.treemanager.option_autolbox_insert = True self.langname = main.name parser.setup_autolbox(main.name, lexer) self.sub = sub self.inserted = 0 self.log = [] def load_main(self, filename): self.filename = filename f = open(filename, "r") self.content = f.read() f.close() self.content.replace("\n", "\r") self.treemanager.import_file(self.content) self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) self.minver = self.treemanager.version def reset(self): self.parser.reset() self.ast = self.parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, self.langname) self.treemanager.import_file(self.content) self.treemanager.option_autolbox_insert = True self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) def load_expr(self, filename): f = open(filename, "r") content = f.read() f.close() self.replexprs = self.find_expressions(content, self.sub_repl_str) def load_expr_from_json(self, filename): import json with open(filename) as f: self.replexprs = json.load(f) def set_replace(self, main, sub): self.main_repl_str = main self.sub_repl_str = sub def find_nonterms_by_name(self, tm, name): l = [] bos = tm.get_bos() eos = tm.get_eos() node = bos.right_sibling() while node is not eos: if validnonterm(node, name): l.append(node) node = next_node(node) continue if node.children: node = node.children[0] else: node = next_node(node) return l def find_expressions(self, program, expr): parser, lexer = self.sub.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, self.sub.name) treemanager.import_file(program) # find all sub expressions l = self.find_nonterms_by_name(treemanager, expr) return [subtree_to_text(st).rstrip() for st in l] def insert_python_expression(self, expr): for c in expr: self.treemanager.key_normal(c) def delete_expr(self, expr): # find first term and last term # select + delete node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[0] else: node = next_node(node) first = node if isinstance(first, MultiTextNode): first = first.children[0] node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[-1] else: node = prev_node(node) while node.lookup == "<ws>" or node.lookup == "<return>": node = node.prev_term last = node if isinstance(last, MultiTextNode): last = last.children[-1] if first.deleted or last.deleted: return None self.treemanager.select_nodes(first, last) deleted = self.treemanager.copySelection() self.treemanager.deleteSelection() return deleted def multi_len(self, autos): r = [] for start, end, _, _ in autos: l = 0 while start is not end: l += len(start.symbol.name) start = start.next_term r.append(l) return r def run(self, main_samples=None, sub_samples=None): assert len(self.treemanager.parsers) == 1 ops = self.main_repl_str, len( [subtree_to_text(x) for x in self.mainexprs]) preversion = self.treemanager.version inserted_error = 0 inserted_valid = 0 noinsert_error = 0 noinsert_valid = 0 noinsert_multi = 0 # pick random exprs from main if not main_samples: samplesize = 10 if len(self.mainexprs) < 10: samplesize = len(self.mainexprs) sample = random.sample(range(len(self.mainexprs)), samplesize) # store this for repeatability exprchoices = [self.mainexprs[i] for i in sample] self.main_samples = sample else: self.main_samples = main_samples exprchoices = [self.mainexprs[i] for i in main_samples] if not sub_samples: # pick random exprs from sub sample = random.sample(range(len(self.replexprs)), len(exprchoices)) replchoices = [self.replexprs[i] for i in sample] self.sub_samples = sample else: self.sub_samples = sub_samples replchoices = [self.replexprs[i] for i in sub_samples] for i in range(len(exprchoices)): e = exprchoices[i] if e.get_root() is None: continue before = len(self.treemanager.parsers) deleted = self.delete_expr(e) mboxes = None ilength = None if deleted: choice = replchoices[i].strip() choice_len = len(choice) if debug: print " Replacing '{}' with '{}':".format( repr(truncate(deleted)), repr(choice)) start = time.time() cursor = self.treemanager.cursor if cursor.node.lookup != "<ws>": # Insert a leading space in situations like `return(x)`. choice = " " + choice if cursor.node.symbol.name[ cursor. pos:] == "" and cursor.node.next_term.lookup != "<ws>": # Insert trailing space if there is none choice = choice + " " self.insert_python_expression(choice) itime = time.time() - start valid = self.parser.last_status if before == len(self.treemanager.parsers): if len( self.parser.error_nodes ) > 0 and self.parser.error_nodes[0].autobox and len( self.parser.error_nodes[0].autobox) > 1: noinsert_multi += 1 result = "No box inserted (Multi)" outcome = "multi" mboxes = self.multi_len( self.parser.error_nodes[0].autobox) elif valid: noinsert_valid += 1 result = "No box inserted (Valid)" outcome = "valid" else: noinsert_error += 1 result = "No box inserted (Error)" outcome = "error" else: result = "Box inserted" innervalid = self.treemanager.parsers[-1][0].last_status self.inserted += 1 recent_box = self.treemanager.parsers[-1][ 0].previous_version.parent lbox_len = self.lbox_length(recent_box) ilength = (lbox_len, choice_len) if valid and innervalid: inserted_valid += 1 outcome = "ok" else: inserted_error += 1 outcome = "inerr" if debug: print " => {} ({})".format(result, valid) nlboxes = len(self.treemanager.parsers) self.log.append([ self.filename, repr(deleted), repr(choice), outcome, itime, mboxes, ilength, nlboxes ]) else: if debug: print "Replacing '{}' with '{}':\n => Already deleted".format( truncate(subtree_to_text(e)), truncate(choice)) self.undo(self.minver) exprchoices = [self.mainexprs[i] for i in self.main_samples] if debug: print("Boxes inserted: {}/{}".format(self.inserted, ops)) print("Valid insertions:", inserted_valid) print("Invalid insertions:", inserted_error) print("No insertion (valid):", noinsert_valid) print("No insertion (error):", noinsert_error) print("No insertion (multi):", noinsert_multi) return (inserted_valid, inserted_error, noinsert_valid, noinsert_error, noinsert_multi) def undo(self, version): # reset everything self.reset() return while self.treemanager.version != version: before = self.treemanager.version self.treemanager.version -= 1 self.treemanager.recover_version("undo", self.treemanager.version + 1) self.treemanager.cursor.load(self.treemanager.version, self.treemanager.lines) if before == self.treemanager.version: exit("Error") def lbox_length(self, root): l = [] node = root.children[0] eos = root.children[-1] while node is not eos: if not node.deleted: l.append(node.symbol.name) node = node.next_term s = "".join(l).strip() return len(s)
class BootstrapParser(object): def __init__(self, lr_type=1, whitespaces=False): self.lr_type = lr_type self.whitespaces = whitespaces # load (old) parser for grammar grammar self.rules = {} self.lrules = [] self.start_symbol = None self.incparser = None self.inclexer = None self.terminals = set() self.extra_alternatives = {} self.change_startrule = None self.options = {"nowhitespace": []} self.precedences = [] self.current_rulename = "" self.all_terminals = set() self.functions = [] self.prod_ids = {} def implicit_ws(self): if self.options.has_key("implicit_ws"): if self.options["implicit_ws"] == "true": return True return False def implicit_newlines(self): if self.options.has_key("implicit_newlines"): if self.options["implicit_newlines"] == "false": return False return True def indentation_based(self): if self.options.has_key("indentation"): if self.options["indentation"] == "true": return True return False def parse(self, ecogrammar): # this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco)) from grammars.eco_grammar import eco_grammar as grammar self.lexer = IncrementalLexer(grammar.priorities) self.parser = IncParser(grammar.grammar, 1, True) self.parser.init_ast() self.ast = self.parser.previous_version.parent self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, grammar.name) self.treemanager.import_file(ecogrammar) if self.parser.last_status == False: raise Exception("Invalid input grammar due to syntax errors") self.read_options() self.parse_both() self.create_parser() self.create_lexer() def parse_both(self): # parse rules startrule = self.ast.children[1] # startrule grammar = startrule.children[1] parser = grammar.children[0] assert parser.symbol.name == "parser" self.parse_rules(parser) # parse lexer startrule = self.ast.children[1] # startrule grammar = startrule.children[1] for element in grammar.children: if element.symbol.name == "lexer": break lexer = element assert lexer.symbol.name == "lexer" self.parse_lexer(lexer) for name, regex in self.lrules: # collect terminals for parser modifications self.all_terminals.add(name) def read_options(self): startrule = self.ast.children[1] # startrule assert startrule.symbol.name == "Startrule" grammar = startrule.children[1] assert grammar.symbol.name == "grammar" for element in grammar.children: if element.symbol.name == "options": break if element.symbol.name != "options": # grammar has no options print("warning: grammar has no options") # backwards compatibility if self.whitespaces: self.options["implicit_ws"] = "true" return options = element assert options.symbol.name == "options" self.parse_options(options) def parse_options(self, options): if options.children == []: return if len(options.children) > 0: assert options.children[0].symbol.name == "settings" self.parse_settings(options.children[0]) if len(options.children) > 1: assert options.children[1].symbol.name == "precedences" self.parse_precedences(options.children[1]) def parse_settings(self, options): if options.children == []: return if len(options.children) == 2: more = options.children[0] self.parse_settings(more) option = options.children[1] else: option = options.children[0] name = option.children[2].symbol.name choice = option.children[6] assert choice.symbol.name == "choice" if choice.children[0].symbol.name == "choice_list": self.options[name] = self.parse_choicelist(choice.children[0]) else: self.options[name] = choice.children[0].symbol.name def parse_choicelist(self, symbol): s = [] for c in symbol.children: if c.symbol.name == ",": continue if c.symbol.name == "WS": continue if c.lookup == "nonterminal": s.append(c.symbol.name) continue if c.symbol.name == "choice_list": rec_s = self.parse_choicelist(symbol.children[0]) s.extend(rec_s) continue return s def parse_precedences(self, precedences): if precedences.children == []: return # recursively parse other precedences if len(precedences.children) == 2: more = precedences.children[0] self.parse_precedences(more) precedence = precedences.children[1] else: precedence = precedences.children[0] # parse single precedence name = precedence.children[0].symbol.name terminals = self.parse_precedence_symbols(precedence.children[2]) self.precedences.append((name, terminals)) def parse_precedence_symbols(self, symbol): s = [] for c in symbol.children: if c.symbol.name == "WS": continue if c.symbol.name == "terminals": rec_s = self.parse_precedence_symbols(symbol.children[0]) s.extend(rec_s) if c.lookup == "terminal": s.append(c.symbol.name[1:-1]) return s def create_parser(self, pickle_id=None): self.all_terminals.update(self.terminals) for fname, terminals, parentrule in self.functions: if fname.startswith("*match_until"): if Nonterminal(fname) not in self.rules: r = Rule(Nonterminal(fname)) for t in self.all_terminals: if t not in terminals: r.add_alternative( [Nonterminal(fname), Terminal(t)], None, t) r.add_alternative([]) self.rules[r.symbol] = r # remove whitespace before special rule from parent rule, e.g. # multistring ::= "MLS" WS *match_until "MLS" WS # ^ this WS causes shift/reduce conflicts prule = self.rules[Nonterminal(parentrule)] for a in prule.alternatives: for i in range(len(a)): sym = a[i] if sym.name == "WS": if len(a) > i + 1 and a[i + 1].name.startswith( "*match_until"): a.pop(i) break if self.implicit_ws(): ws_rule = Rule() ws_rule.symbol = Nonterminal("WS") ws_rule.add_alternative([Nonterminal("WS"), Terminal("<ws>")]) # get comment rule if self.options.has_key('comment_rule'): cmt_rules = self.options['comment_rule'] for cmt_rule in cmt_rules: if Nonterminal(cmt_rule) in self.rules: ws_rule.add_alternative( [Nonterminal("WS"), Nonterminal("comment")]) if self.implicit_newlines(): ws_rule.add_alternative( [Nonterminal("WS"), Terminal("<return>")]) ws_rule.add_alternative([ Nonterminal("WS"), Terminal("<backslash>"), Terminal("<return>") ]) ws_rule.add_alternative([]) # or empty self.rules[ws_rule.symbol] = ws_rule for a in ws_rule.alternatives: self.prod_ids[Production(ws_rule.symbol, a)] = len(self.prod_ids) # allow whitespace/comments at beginning of file start_rule = Rule() start_rule.symbol = Nonterminal("Startrule") start_rule.add_alternative([Nonterminal("WS"), self.start_symbol]) self.rules[start_rule.symbol] = start_rule self.prod_ids[Production(start_rule.symbol, start_rule.alternatives[0])] = len( self.prod_ids) self.start_symbol = start_rule.symbol incparser = IncParser() incparser.from_dict(self.rules, self.start_symbol, self.lr_type, self.implicit_ws(), pickle_id, self.precedences, self.prod_ids) incparser.init_ast() self.incparser = incparser def parse_rules(self, node): if node.children[0].symbol.name == "parser": self.parse_rules(node.children[0]) self.parse_rule(node.children[3]) elif node.children[0].symbol.name == "rule": self.parse_rule(node.children[0]) def parse_rule(self, node): name = node.children[0].symbol.name self.current_rulename = name alternatives = self.parse_alternatives(node.children[4]) symbol = Nonterminal(name) if self.start_symbol is None: self.start_symbol = symbol if self.change_startrule and symbol.name == self.change_startrule: self.start_symbol = symbol r = Rule(symbol) for a in alternatives: r.add_alternative(a[0], a[1], a[2]) self.prod_ids[Production(symbol, a[0])] = len(self.prod_ids) # add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes) if self.extra_alternatives.has_key(symbol.name): for n in self.extra_alternatives[symbol.name]: a = [MagicTerminal(n), Nonterminal("WS")] r.add_alternative(a) self.prod_ids[Production(symbol, a)] = len(self.prod_ids) self.rules[symbol] = r def parse_alternatives(self, node): if node.children[0].symbol.name == "alternatives": alternatives = self.parse_alternatives(node.children[0]) alternative = self.parse_alternative(node.children[3]) alternatives.append(alternative) return alternatives elif node.children[0].symbol.name == "right": return [self.parse_alternative(node.children[0])] def parse_alternative(self, node): if len(node.children) > 0: annotation = None prec = None for c in node.children: if c.symbol.name == "symbols": symbols = self.parse_symbols(c) if c.symbol.name == "prec": prec = self.parse_prec(c) if c.symbol.name == "annotations": annotation = self.parse_annotation(c) return (symbols, annotation, prec) else: return ([], None, None) def parse_prec(self, node): if node.children: c = node.children[2] return c.symbol.name[1:-1] def parse_symbols(self, node): if node.children[0].symbol.name == "symbols": symbols = self.parse_symbols(node.children[0]) symbol = self.parse_symbol(node.children[1]) symbols.append(symbol) if ( isinstance(symbol, Terminal) or isinstance(symbol, MagicTerminal) ) and self.implicit_ws( ) and self.current_rulename not in self.options["nowhitespace"]: symbols.append(Nonterminal("WS")) return symbols elif node.children[0].symbol.name == "symbol": l = [] symbol = self.parse_symbol(node.children[0]) l.append(symbol) if isinstance(symbol, Terminal) and self.implicit_ws( ) and self.current_rulename not in self.options["nowhitespace"]: l.append(Nonterminal("WS")) return l def parse_symbol(self, node): node = node.children[0] if node.lookup == "nonterminal": return Nonterminal(node.symbol.name) elif node.lookup == "terminal": if node.symbol.name != "\"<eos>\"": self.terminals.add(node.symbol.name[1:-1]) return Terminal(node.symbol.name[1:-1]) elif node.lookup == "languagebox": return MagicTerminal(node.symbol.name) elif node.symbol.name == "function": return self.parse_function(node) def parse_function(self, node): fname = node.children[0].symbol.name terminals = self.parse_fargs(node.children[4]) safe_name = "*%s%s" % (fname, hash(frozenset(terminals))) self.functions.append((safe_name, terminals, self.current_rulename)) return Nonterminal(safe_name) def parse_fargs(self, symbol): s = [] for c in symbol.children: if c.symbol.name == ",": continue if c.symbol.name == "WS": continue if c.lookup == "terminal": s.append(c.symbol.name[1:-1]) continue if c.symbol.name == "f_args": rec_s = self.parse_fargs(symbol.children[0]) s.extend(rec_s) return s def parse_annotation(self, node): a_options = node.children[2] assert a_options.symbol.name == "a_options" if a_options.children[0].symbol.name == "astnode": return self.parse_astnode(a_options.children[0]) elif a_options.children[0].symbol.name == "expression": return self.parse_expression(a_options.children[0]) elif a_options.children[0].symbol.name == "forloop": return self.parse_foreach(a_options.children[0]) def parse_astnode(self, node): name = node.children[0].symbol.name children = self.parse_astnode_children(node.children[4]) d = {} for n, expr in children: d[n] = expr return AstNode(name, d) def parse_astnode_children(self, node): assert node.symbol.name == "astnode_children" if node.children[0].symbol.name == "astnode_child": return [self.parse_astnode_child(node.children[0])] elif node.children[0].symbol.name == "astnode_children": children = self.parse_astnode_children(node.children[0]) child = self.parse_astnode_child(node.children[3]) children.append(child) return children def parse_astnode_child(self, node): assert node.symbol.name == "astnode_child" name = node.children[0].symbol.name if node.children[4].symbol.name == "expression": expr = self.parse_expression(node.children[4]) elif node.children[4].symbol.name == "reference": expr = self.parse_reference(node.children[4]) return (name, expr) def parse_expression(self, node): if node.children[0].symbol.name == "node": return self.parse_node(node.children[0]) elif node.children[0].symbol.name == "list": return self.parse_list(node.children[0]) elif node.children[0].symbol.name == "node_ref": return self.parse_noderef(node.children[0]) else: expr1 = self.parse_expression(node.children[0]) if node.children[3].symbol.name == "node": expr2 = self.parse_node(node.children[3]) else: expr2 = self.parse_list(node.children[3]) return AddExpr(expr1, expr2) def parse_foreach(self, node): item = self.parse_node(node.children[4]) expr = self.parse_astnode(node.children[7]) return Foreach(node.symbol.name, item, expr) def parse_noderef(self, node): lookup = self.parse_node(node.children[0]) attr = node.children[3] lookup.attribute = attr.symbol.name return lookup def parse_node(self, node): return LookupExpr(int(node.children[2].symbol.name)) def parse_list(self, node): return ListExpr(self.parse_listloop(node.children[2])) def parse_reference(self, node): base = node.children[0].symbol.name ref = node.children[4].symbol.name return ReferenceExpr(base, ref) def parse_listloop(self, node): if len(node.children) == 0: return [] if node.children[0].symbol.name == "list_loop": l = self.parse_listloop(node.children[0]) element = self.parse_unknown(node.children[3]) l.append(element) return l else: return [self.parse_unknown(node.children[0])] def parse_unknown(self, node): if node.symbol.name == "node": return self.parse_node(node) elif node.symbol.name == "astnode": return self.parse_astnode(node) def create_lexer(self, buildlexer=True): names = [] regexs = [] for name, regex in self.lrules: names.append(name) self.all_terminals.add(name) regexs.append(regex) # add so far undefined terminals undefined_terminals = self.terminals.difference(set(names)) import re for t in undefined_terminals: names.insert(0, t) regexs.insert(0, re.escape(t)) if not buildlexer: self.inclexer = (names, regexs) return self.inclexer = IncrementalLexerCF() self.inclexer.from_name_and_regex(names, regexs) if self.indentation_based(): self.inclexer.indentation_based = True def parse_lexer(self, lexer): if lexer.children[0].symbol.name == "lrule": self.parse_lrule(lexer.children[0]) elif lexer.children[0].symbol.name == "lexer": self.parse_lexer(lexer.children[0]) self.parse_lrule(lexer.children[1]) def parse_lrule(self, lrule): assert lrule.children[0].symbol.name == "tokenname" name = lrule.children[0].children[0].symbol.name regex = lrule.children[3].symbol.name[1:-1] self.lrules.append((name, regex))
class Test_MultiTextNodePython: def setup_class(cls): parser, lexer = python.load() cls.lexer = lexer cls.parser = parser cls.parser.init_ast() cls.ast = cls.parser.previous_version cls.treemanager = TreeManager() cls.treemanager.add_parser(cls.parser, cls.lexer, python.name) cls.treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite def reset(self): self.parser.reset() self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, calc.name) self.treemanager.set_font_test(7, 17) def test_simple(self): self.reset() inputstring = "x = \"\"\"abcdef\"\"\"" for c in inputstring: self.treemanager.key_normal(c) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.key_normal("\r") def test_relex_over_indentation(self): self.reset() inputstring = """class X: x = 1 def x(): pass y = 2""" self.treemanager.import_file(inputstring) assert self.parser.last_status == True self.treemanager.cursor_movement(DOWN) self.treemanager.cursor_movement(DOWN) self.treemanager.cursor_movement(DOWN) self.treemanager.key_end() assert self.treemanager.cursor.node.symbol.name == "pass" self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.cursor_movement(UP) self.treemanager.cursor_movement(UP) self.treemanager.key_end() assert self.treemanager.cursor.node.symbol.name == "1" self.treemanager.cursor_movement(LEFT) self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") assert self.parser.last_status == True def test_indentation_to_string_and_back(self): self.reset() inputstring = """class X: a b""" self.treemanager.import_file(inputstring) assert self.parser.last_status == True self.treemanager.cursor_movement(DOWN) self.treemanager.cursor_movement(DOWN) self.treemanager.key_end() self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.cursor_movement(UP) self.treemanager.cursor_movement(UP) self.treemanager.key_home() self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") self.treemanager.key_normal("\"") assert self.parser.last_status == True self.treemanager.cursor_movement(DOWN) self.treemanager.cursor_movement(DOWN) self.treemanager.key_end() self.treemanager.key_backspace() assert self.parser.last_status == False def test_remember_open_lexing_states(self): self.reset() inputstring = """x = 1 y = 2""" self.treemanager.import_file(inputstring) assert self.parser.last_status == True self.treemanager.key_end() self.treemanager.cursor_movement(LEFT) self.treemanager.key_normal("\"") #assert self.parser.last_status == False # unfinished lexing jobs self.treemanager.key_end() self.treemanager.key_normal("\"") assert self.parser.last_status == True def test_triplequote_string(self): self.reset() inputstring = 'x="""abc"""' for i in inputstring: self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""abc"""' self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.key_normal("\"") assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '""' assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"' assert bos.next_term.next_term.next_term.next_term.lookback == 1 assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == 'c' assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2 assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""' assert bos.next_term.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"' self.treemanager.key_normal("\"") assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '""' assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"' assert bos.next_term.next_term.next_term.next_term.lookback == 1 assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '"c"' assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2 assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""' self.treemanager.key_normal("\"") #assert self.parser.last_status == False assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""' assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c' assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""' assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"' self.treemanager.key_end() self.treemanager.key_backspace() assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""' assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c' assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""' def test_ignore_nonlbox_x80(self): self.reset() inputstring = 'x="""ab\x80c"""' for i in inputstring: self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""ab\x80c"""' def test_multinode_from_the_start(self): self.reset() inputstring = '''x="""a\rbc"""''' for i in inputstring: self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>' def test_multinode_and_nonlbox_x80(self): self.reset() inputstring = '''x="""a\x80bc"""''' for i in inputstring: self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""' self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.key_normal("\r") bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>' def test_multinode_nonlbox_and_lbox(self): self.reset() inputstring = '''x="""a\x80bc"""''' for i in inputstring: self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""' self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.add_languagebox(lang_dict["SQL"]) self.treemanager.key_normal("S") bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "x" assert bos.next_term.next_term.symbol.name == "=" assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>' multi = bos.next_term.next_term.next_term assert multi.children[0].symbol.name == "\"\"\"a\x80b" assert type(multi.children[1].symbol) is MagicTerminal assert multi.children[2].symbol.name == "c\"\"\"" def test_multinode_merged_first(self): self.reset() inputstring = '''"""a\rbc"""''' for i in inputstring: self.treemanager.key_normal(i) for i in 'def"""': self.treemanager.key_normal(i) bos = self.parser.previous_version.parent.children[0] assert bos.next_term.symbol.name == "<Multinode>" assert bos.next_term.next_term.symbol.name == 'def' assert bos.next_term.next_term.next_term.symbol.name == '""' assert bos.next_term.next_term.next_term.next_term.symbol.name == '"' self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.key_backspace() self.treemanager.key_backspace() self.treemanager.key_backspace() assert bos.next_term.symbol.name == "<Multinode>" assert bos.next_term.next_term.symbol.name == "NEWLINE" assert bos.next_term.next_term.next_term.symbol.name == "eos" def test_multinode_string_bug(self): self.reset() inputstring = '''x="abc"''' for i in inputstring: self.treemanager.key_normal(i) self.treemanager.cursor_movement(LEFT) self.treemanager.add_languagebox(lang_dict["SQL"]) self.treemanager.key_normal("x") bos = self.parser.previous_version.parent.children[0] x = bos.next_term assert x.symbol.name == "x" eq = x.next_term assert eq.symbol.name == "=" multi = eq.next_term assert multi.lookup == "dstring" assert multi.symbol.name == "<Multinode>" self.treemanager.cursor_movement(RIGHT) self.treemanager.key_backspace() # removing the ending quote results in a lexingerror, # so the multinode remains assert eq.next_term.symbol.name == "<Multinode>" # now remove the first quote, which should lead to the destruction of # the multinode self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.cursor_movement(LEFT) self.treemanager.key_backspace() assert eq.next_term.symbol.name == "abc" def test_multinode_string_bug2(self): self.reset() inputstring = '''x="abc"''' for i in inputstring: self.treemanager.key_normal(i) self.treemanager.cursor_movement(LEFT) self.treemanager.add_languagebox(lang_dict["SQL"]) self.treemanager.key_normal("x") self.treemanager.leave_languagebox() self.treemanager.key_normal("z") bos = self.parser.previous_version.parent.children[0] x = bos.next_term assert x.symbol.name == "x" eq = x.next_term assert eq.symbol.name == "=" multi = eq.next_term assert multi.children[0].symbol.name == "\"abc" assert multi.children[1].symbol.name == "<SQL>" assert multi.children[2].symbol.name == "z\""
whitespace = True lexer = IncrementalLexer(grammar.priorities) parser = IncParser(grammar.grammar, 1, whitespace) parser.init_ast() ast = parser.previous_version treemanager = TreeManager() treemanager.add_parser(parser, lexer, grammar.name) treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite inputstring = """import abc.xyz as efg from x import z class Test: def x(): if x == 1: z = 3 + 4 * 5 elif x == 2: for x in range(2,10): print x else: z = 4 return 1 if a==b else 2""" treemanager.import_file(inputstring) viewer = Viewer("pydot") viewer.get_tree_image(treemanager.get_mainparser().previous_version.parent, [], whitespace, ast=True) import os os.system("xdg-open " + viewer.image)
class NodeEditor(QFrame): # ========================== init stuff ========================== # def __init__(self, parent=None): QtGui.QWidget.__init__(self, parent) self.viewport_y = 0 # top visible line self.imagemode = False self.image = None self.scroll_height = 0 self.scroll_width = 0 self.timer = QTimer(self) self.backuptimer = QTimer(self) self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer) self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer) self.backuptimer.start(30000) self.undotimer = QTimer(self) self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer) self.blinktimer = QTimer(self) self.blinktimer.start(500) self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer) self.show_cursor = True self.boxcolors = [QColor("#DC322F"), QColor("#268BD2"), QColor("#2Ac598"), QColor("#D33682"), QColor("#B58900"), QColor("#859900")] self.setCursor(Qt.IBeamCursor) # Semi-transparent overlay. # Used to display heat-map visualisation of profiler info, etc. self.overlay = Overlay(self) # Always show the overlay, users hide visualisations with the HUD. self.overlay.show() # Show / don't show HUD visualisations. self.hud_callgraph = False self.hud_eval = False self.hud_heat_map = False self.hud_types = False def hud_show_callgraph(self): self.hud_callgraph = True self.hud_eval = False self.hud_heat_map = False self.hud_types = False def hud_show_eval(self): self.hud_callgraph = False self.hud_eval = True self.hud_heat_map = False self.hud_types = False def hud_show_types(self): self.hud_callgraph = False self.hud_eval = False self.hud_heat_map = False self.hud_types = True def hud_show_heat_map(self): self.hud_callgraph = False self.hud_eval = False self.hud_heat_map = True self.hud_types = False def hud_off(self): self.hud_callgraph = False self.hud_eval = False self.hud_heat_map = False self.hud_types = False def focusOutEvent(self, event): self.blinktimer.stop() self.show_cursor = True self.update() def focusInEvent(self, event): self.blinktimer.start() def resizeEvent(self, event): self.overlay.resize(event.size()) event.accept() def analysis_timer(self): if self.getWindow().show_namebinding(): self.tm.analyse() self.update() self.timer.stop() # save swap filename = self.getEditorTab().filename if filename: self.saveToJson(filename + ".swp", True) def backup_timer(self): filename = self.getEditorTab().filename if filename: self.saveToJson(filename + ".bak", True) def trigger_blinktimer(self): if self.timer.isActive(): self.show_cursor = True return self.show_cursor ^= True self.update() def trigger_undotimer(self): self.tm.undo_snapshot() self.undotimer.stop() def setImageMode(self, boolean): self.imagemode = boolean def reset(self): self.update() def set_mainlanguage(self, parser, lexer, lang_name): self.tm = TreeManager() self.tm.add_parser(parser, lexer, lang_name) def get_mainlanguage(self): return self.tm.parsers[0][2] def set_sublanguage(self, language): self.sublanguage = language def event(self, event): if event.type() == QEvent.ToolTip: pos = event.pos() temp_cursor = self.tm.cursor.copy() result = self.coordinate_to_cursor(pos.x(), pos.y()) node = self.tm.cursor.node self.tm.cursor.line = temp_cursor.line self.tm.cursor.node = temp_cursor.node self.tm.cursor.pos = temp_cursor.pos if not result: QToolTip.hideText() event.ignore() return True # Draw errors, if there are any. msg = self.tm.get_error(node) if msg: QToolTip.showText(event.globalPos(), msg) return True # Draw tooltips if there are any. Tooltips can appear with any HUD # visualisation. elif (self.hud_callgraph or self.hud_eval or self.hud_types or self.hud_heat_map): strings = [] annotes = node.get_annotations_with_hint(ToolTip) if self.hud_callgraph: for annote in annotes: if annote.has_hint(HUDCallgraph): strings.append(annote.annotation) elif self.hud_eval: for annote in annotes: if annote.has_hint(HUDEval): strings.append(annote.annotation) elif self.hud_types: for annote in annotes: if annote.has_hint(HUDTypes): strings.append(annote.annotation) elif self.hud_heat_map: for annote in annotes: if annote.has_hint(HUDHeatmap): strings.append(annote.annotation) msg = "\n".join(strings) if msg.strip() != "": if self.tm.tool_data_is_dirty: msg += "\n[Warning: Information may be out of date.]" QToolTip.showText(event.globalPos(), msg) return True QToolTip.hideText() event.ignore() return True return QFrame.event(self, event) # ========================== GUI related stuff ========================== # def blink(self): if self.show_cursor: self.show_cursor = 0 else: self.show_cursor = 1 self.update() def sliderChanged(self, value): self.viewport_y = value self.overlay.start_line = self.viewport_y self.update() def sliderXChanged(self, value): self.move(-value*self.fontwt,0) self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height()) if self.x() == 0: self.updateGeometry() self.update() def getScrollSizes(self): total_lines = 0 max_width = 0 for l in self.lines: total_lines += l.height max_width = max(max_width, l.width) max_visible_lines = self.geometry().height() / self.fontht self.scroll_height = max(0, total_lines - max_visible_lines) current_width = self.parentWidget().geometry().width() / self.fontwt self.scroll_width = max(0, max_width - current_width) def paintEvent(self, event): # Clear data in the visualisation overlay self.overlay.clear_data() gfont = QApplication.instance().gfont self.font = gfont.font self.fontwt = gfont.fontwt self.fontht = gfont.fontht self.fontd = gfont.fontd QtGui.QFrame.paintEvent(self, event) paint = QtGui.QPainter() if self.imagemode: self.image = QImage() paint.begin(self.image) else: paint.begin(self) paint.setFont(self.font) self.longest_column = 0 # calculate how many lines we need to show self.init_height = self.geometry().height() self.paintLines(paint, self.viewport_y) paint.end() total_lines = 0 max_width = 0 for l in self.lines: total_lines += l.height max_width = max(max_width, l.width) max_visible_lines = self.geometry().height() / self.fontht self.scroll_height = max(0, total_lines - max_visible_lines) current_width = self.parentWidget().geometry().width() / self.fontwt self.scroll_width = max(0, max_width - current_width) railroad_annotations = self.tm.get_all_annotations_with_hint(Railroad) self.overlay.add_railroad_data(railroad_annotations) self.emit(SIGNAL("painted()")) # paint lines using new line manager def paintLines(self, paint, startline): # find internal line corresponding to visual line visual_line = 0 internal_line = 0 for l in self.tm.lines: if visual_line + l.height > startline: break visual_line += l.height internal_line += 1 x = 0 y = visual_line - startline # start drawing outside of viewport to display partial images self.paint_start = (internal_line, y) max_y = self.geometry().height() / self.fontht line = internal_line node = self.tm.lines[line].node _, _, self.end_line = self.paint_nodes(paint, node, x, y, line, max_y) #XXX if starting node is inside language box, init lbox with amount of language boxes def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0): settings = QSettings("softdev", "Eco") colors = self.boxcolors if settings.value("app_theme", "Light").toString() in ["Dark"]: alpha = 100 self.highlight_line_color = QColor(250,250,250,20) elif settings.value("app_theme", "Light").toString() in ["Gruvbox"]: alpha = 100 self.highlight_line_color = QColor(250,250,250,20) else: alpha = 60 self.highlight_line_color = QColor(0,0,0,10) self.show_highlight_line = settings.value("highlight_line", False).toBool() first_node = node selected_language = self.tm.mainroot error_node = self.tm.get_mainparser().error_node error_node = self.fix_errornode(error_node) highlighter = self.get_highlighter(node) selection_start = min(self.tm.selection_start, self.tm.selection_end) selection_end = max(self.tm.selection_start, self.tm.selection_end) draw_selection_start = (0,0,0) draw_selection_end = (0,0,0) start_lbox = self.get_languagebox(node) renderer = self.get_renderer(node) self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node) #XXX get initial x for langbox if start_lbox: lbox += 1 if start_lbox and self.selected_lbox is start_lbox: draw_lbox = True else: draw_lbox = False draw_all_boxes = self.getWindow().show_languageboxes() self.lines = self.tm.lines self.cursor = self.tm.cursor self.lines[line].height = 1 # reset height draw_cursor = True show_namebinding = self.getWindow().show_namebinding() while y < max_y: # if we found a language box, continue drawing inside of it if isinstance(node.symbol, MagicTerminal): lbox += 1 lbnode = node.symbol.ast if self.selected_lbox is node: color = colors[(lbox-1) % len(colors)] self.draw_lbox_bracket(paint, '[', node, x, y, color) draw_lbox = True selected_language = lbnode else: draw_lbox = False node = lbnode.children[0] highlighter = self.get_highlighter(node) renderer = self.get_renderer(node) error_node = self.tm.get_parser(lbnode).error_node error_node = self.fix_errornode(error_node) continue if isinstance(node, EOS): lbnode = self.get_languagebox(node) if self.cursor.node is lbnode: self.draw_cursor(paint, x, 4 + y * self.fontht) if lbnode: color = colors[(lbox-1) % len(colors)] if lbox > 0: lbox -= 1 node = lbnode.next_term highlighter = self.get_highlighter(node) renderer = self.get_renderer(node) if self.selected_lbox is lbnode: # draw bracket self.draw_lbox_bracket(paint, ']', node, x, y, color) draw_lbox = False lbnode = self.get_languagebox(node) if lbnode and self.selected_lbox is lbnode: draw_lbox = True error_node = self.tm.get_parser(lbnode.symbol.ast).error_node error_node = self.fix_errornode(error_node) else: error_node = self.tm.get_mainparser().error_node error_node = self.fix_errornode(error_node) continue else: self.lines[line].width = x / self.fontwt break # draw language boxes if lbox > 0 and (draw_lbox or draw_all_boxes): if draw_all_boxes: color = colors[(lbox-1) % len(colors)] color.setAlpha(alpha) else: color = colors[0] color.setAlpha(alpha) if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box color.setAlpha(20) renderer.update_image(node) if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal): if not node.image or node.plain_mode: if isinstance(node, BOS) and isinstance(node.next_term, EOS): self.draw_lbox_hints(paint, node, x, y, color) else: paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color) # prepare selection drawing if node is selection_start.node: if node.lookup == "<return>": sel_x = x else: sel_x = x + selection_start.pos * self.fontwt draw_selection_start = (sel_x, y, line) if node is selection_end.node: draw_selection_end = (x + selection_end.pos * self.fontwt, y, line) # draw node dx, dy = renderer.paint_node(paint, node, x, y, highlighter) x += dx self.lines[line].height = max(self.lines[line].height, dy) # Draw footnotes and add heatmap data to overlay. annotes = [annote.annotation for annote in node.get_annotations_with_hint(Heatmap)] # Heatmap data can always be sent to the overlay, it won't be # rendered unless the user selects the Heatmap HUD radio button. for annote in annotes: self.overlay.add_heatmap_datum(line + 1, annote) if self.hud_eval or self.hud_types: infofont = QApplication.instance().tool_info_font annotes = [] annotes_ = node.get_annotations_with_hint(Footnote) if self.hud_eval: for annote in annotes_: if annote.has_hint(HUDEval): annotes.append(annote) elif self.hud_types: for annote in annotes_: if annote.has_hint(HUDTypes): annotes.append(annote) footnote = " ".join([annote.annotation for annote in annotes]) if footnote.strip() != "": if not self.tm.tool_data_is_dirty: infofont.font.setBold(True) else: infofont.font.setBold(False) paint.setFont(infofont.font) paint.setPen(QPen(QColor((highlighter.get_default_color())))) start_y = self.fontht + ((y + 1) * self.fontht) paint.drawText(QtCore.QPointF(x-dx, start_y), footnote) self.lines[line].height = max(self.lines[line].height, 2) paint.setFont(self.font) # after we drew a return, update line information if node.lookup == "<return>" and not node is first_node: # draw lbox to end of line if draw_lbox or (draw_all_boxes and lbox > 0): paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color) self.lines[line].width = x / self.fontwt x = 0 y += self.lines[line].height line += 1 self.lines[line].height = 1 # reset height if self.show_highlight_line: if node.lookup == "<return>" or isinstance(node, BOS): if self.cursor.line == line: self.highlight_line(paint, y) # draw cursor if node is self.cursor.node and self.show_cursor: draw_x = max(0, x-dx) cursor_pos = self.cursor.pos if node.symbol.name == "\r": cursor_pos = 0 if node.image and not node.plain_mode: draw_x = x cursor_pos = 0 self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 4 + y * self.fontht) if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor: draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3) paint.drawRect(draw_cursor_at) # set lbox info coordinates infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht) draw_cursor = False # draw squiggly line if node is error_node or (show_namebinding and self.tm.has_error(node)): if isinstance(node, EOS): length = self.fontwt else: length = len(node.symbol.name)*self.fontwt if isinstance(node.symbol, MagicTerminal): self.draw_vertical_squiggly_line(paint,x,y) else: if self.tm.has_error(node): err_color = "orange" else: err_color = "red" self.draw_squiggly_line(paint,x-length,y,length, err_color) node = node.next_term if selection_start != selection_end: self.draw_selection(paint, draw_selection_start, draw_selection_end, max_y) # paint infobox if False: lang_name = self.tm.parsers[selected_language] lang_status = self.tm.get_parser[selected_language][0].last_status if lang_status is True: color = QColor(100,255,100) else: color = QColor(255,100,100) paint.setFont(infofont) paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*infofontwt, infofontht), color) paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name) paint.setFont(self.font) return x, y, line def highlight_line(self, paint, y): width = self.parentWidget().geometry().width() paint.fillRect(QRect(-5, y * self.fontht + 3, width, self.fontht), self.highlight_line_color) def draw_lbox_hints(self, paint, node, x, y, color): if node is self.cursor.node: return alpha = color.alpha() color.setAlpha(255) path = QPainterPath() x = x - 2 y = y*self.fontht + 4 path.moveTo(x, y) path.lineTo(x+6, y) path.lineTo(x+3, y+3) path.lineTo(x, y) paint.fillPath (path, QBrush (color)); color.setAlpha(alpha) def draw_lbox_bracket(self, paint, bracket, node, x, y, color): assert bracket in ['[',']'] oldpen = paint.pen() newpen = QPen() color.setAlpha(255) newpen.setColor(color) newpen.setWidth(1) paint.setPen(newpen) # paint brackets path = QPainterPath() if bracket == '[': if x == 0: tmpx = x + 2 else: tmpx = x + 1 # adjust bracket position path.moveTo(tmpx, 3+y*self.fontht) path.lineTo(tmpx-2, 3+y*self.fontht) path.moveTo(tmpx-2, 3+y*self.fontht) path.lineTo(tmpx-2, 3+y*self.fontht + self.fontht - 1) else: tmpx = x - 1 path.moveTo(tmpx, 3+y*self.fontht) path.lineTo(tmpx+2, 3+y*self.fontht) path.moveTo(tmpx+2, 3+y*self.fontht) path.lineTo(tmpx+2, 3+y*self.fontht + self.fontht - 1) path.lineTo(tmpx, 3+y*self.fontht + self.fontht - 1) paint.drawPath(path) paint.setPen(oldpen) def fix_errornode(self, error_node): if not error_node: return while isinstance(error_node.symbol, IndentationTerminal): error_node = error_node.next_term return error_node def draw_cursor(self, paint, x, y): pen = paint.pen() colorhex = self.palette().color(QPalette.Text) pen.setColor(QColor(colorhex)) paint.setPen(pen) draw_cursor_at = QRect(x, y, 0, self.fontht - 3) paint.drawRect(draw_cursor_at) def draw_vertical_squiggly_line(self, paint, x, y): paint.setPen(Qt.CustomDashLine) pen = paint.pen() pen.setDashPattern([2,2]) pen.setColor(QColor("red")) paint.setPen(pen) y = 3+y*self.fontht paint.drawLine(x-1, y, x-1, y+self.fontht) paint.drawLine(x, y+2, x, y+self.fontht) paint.setPen(Qt.SolidLine) def draw_squiggly_line(self, paint, x, y, length, color): paint.setPen(Qt.CustomDashLine) pen = paint.pen() pen.setDashPattern([2,2]) pen.setColor(QColor(color)) paint.setPen(pen) y = (y+1)*self.fontht+1 paint.drawLine(x, y, x+length, y) paint.drawLine(x+2, y+1, x+2+length, y+1) paint.setPen(Qt.SolidLine) def draw_selection(self, paint, draw_selection_start, draw_selection_end, max_y): x1, y1, line1 = draw_selection_start x2, y2, line2 = draw_selection_end start = min(self.tm.selection_start, self.tm.selection_end) end = max(self.tm.selection_start, self.tm.selection_end) if x1 + y1 + line1 + x2 + y2 + line2 == 0: # everything out of viewport, draw nothing # unless start and end are on opposite sides of the viewport if not(start.line <= self.paint_start[0] and end.line >= self.paint_start[0] + max_y): return if x1 + y1 + line1 == 0: # start outside of viewport line1 = self.paint_start[0] if x2 + y2 + line2 == 0: # end outside of viewport line2 = self.paint_start[0] + max_y y2 = max_y if y1 == y2: paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100)) else: width = max(self.fontwt, self.tm.lines[line1].width*self.fontwt) paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, width - x1, self.fontht), QColor(0,0,255,100)) y = y1 + self.tm.lines[line1].height for i in range(line1+1, line2): width = self.tm.lines[i].width*self.fontwt if width == 0: width = self.fontwt paint.fillRect(QRectF(0, 3 + y * self.fontht, width, self.fontht), QColor(0,0,255,100)) y = y + self.tm.lines[i].height paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100)) def get_highlighter(self, node): root = node.get_root() base = lang_dict[self.tm.get_language(root)].base s = syntaxhighlighter.get_highlighter(base, self.palette()) return s def get_languagebox(self, node): root = node.get_root() lbox = root.get_magicterminal() return lbox def get_renderer(self, node): root = node.get_root() base = lang_dict[self.tm.get_language(root)].base return renderers.get_renderer(base, self.fontwt, self.fontht, self.fontd) def focusNextPrevChild(self, b): # don't switch to next widget on TAB return False def mousePressEvent(self, e): if e.button() == Qt.LeftButton: self.tm.input_log.append("# mousePressEvent") self.coordinate_to_cursor(e.x(), e.y()) self.tm.selection_start = self.tm.cursor.copy() self.tm.selection_end = self.tm.cursor.copy() self.tm.input_log.append("self.selection_start = self.cursor.copy()") self.tm.input_log.append("self.selection_end = self.cursor.copy()") self.getWindow().showLookahead() self.update() def mouseDoubleClickEvent(self, e): if e.button() == Qt.LeftButton: self.coordinate_to_cursor(e.x(), e.y()) node = self.tm.get_node_from_cursor() lbox = self.get_languagebox(node) if lbox and lbox.symbol.name == "<IPython>": if lbox.plain_mode is False: lbox.plain_mode = True else: lbox.plain_mode = False self.update() return elif node.image is None: self.tm.doubleclick_select() self.update() return if node.plain_mode is False: node.plain_mode = True self.tm.cursor.pos = len(node.symbol.name) else: node.plain_mode = False self.update() def cursor_to_coordinate(self): y = 0 for l in self.tm.lines[:self.cursor.line]: y += l.height * self.fontht x = self.tm.cursor.get_x() * self.fontwt y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht return (x,y) def coordinate_to_cursor(self, x, y): mouse_y = y / self.fontht first_line = self.paint_start[0] y_offset = self.paint_start[1] y = y_offset line = first_line if mouse_y < 0: while line > 0: y -= self.tm.lines[line].height if y < mouse_y: break line -= 1 else: while line < len(self.tm.lines) - 1: y += self.tm.lines[line].height if y > mouse_y: break line += 1 self.tm.cursor.line = line cursor_x = int(round(float(x) / self.fontwt)) self.tm.cursor.move_to_x(cursor_x) self.tm.input_log.append("self.cursor.line = %s" % str(line)) self.tm.log_input("cursor.move_to_x", str(cursor_x)) if mouse_y > y or self.tm.cursor.get_x() != cursor_x: return False return True def mouseMoveEvent(self, e): # apparently this is only called when a mouse button is clicked while # the mouse is moving if self.tm.input_log[-2].startswith("self.cursor.move_to_x"): # only log the last move event self.tm.input_log.pop() self.tm.input_log.pop() self.tm.input_log.pop() self.coordinate_to_cursor(e.x(), e.y()) self.tm.selection_end = self.tm.cursor.copy() self.tm.input_log.append("self.selection_end = self.cursor.copy()") self.update() self.getEditorTab().keypress() def keyPressEvent(self, e): self.timer.start(500) self.show_cursor = True startundotimer = False key = KeyPress(e) # key presses to ignore if key.is_modifier or key.page_up or key.page_down: return # has been processes in get_nodes_at_pos -> reset self.edit_rightnode = False if key.escape: self.tm.key_escape() elif key.backspace: startundotimer = True self.tm.key_backspace() elif key.home: self.tm.key_home(shift=key.m_shift) elif key.end: self.tm.key_end(shift=key.m_shift) elif key.is_arrow: if key.jump_word: self.tm.ctrl_cursor(key, shift=key.m_shift) else: self.tm.key_cursors(key, shift=key.m_shift) elif key.delete: startundotimer = True self.tm.key_delete() elif e.key() == Qt.Key_F3: self.tm.find_next() # User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't # sensibly insert into the text. elif key.has_action_modifier: pass # every other normal key press else: startundotimer = True if e.key() == Qt.Key_Tab: text = " " else: # text is a char array, so we need the first letter # to match it against the set text = e.text() if text.isEmpty() or text.toUtf8()[0] not in whitelist: logging.debug("Key %s not supported" % text) return self.tm.key_normal(text) self.getWindow().btReparse([]) self.update() self.emit(SIGNAL("keypress(QKeyEvent)"), e) self.getWindow().showLookahead() if startundotimer: self.undotimer.start(500) def showLanguageBoxMenu(self): self.showSubgrammarMenu() self.create_languagebox() def create_languagebox(self): if self.sublanguage: if self.tm.hasSelection(): self.tm.surround_with_languagebox(self.sublanguage) else: self.tm.add_languagebox(self.sublanguage) def change_languagebox(self): if self.sublanguage: self.tm.change_languagebox(self.sublanguage) def showCodeCompletion(self): l = self.tm.getCompletion() if l: self.showCodeCompletionMenu(l) def println(self, prestring, y): node = self.lines[y].node.next_term x = [] while node is not None and node.symbol.name != "\r": x.append(node.symbol.name) node = node.next_term print(prestring, "".join(x)) def print_line(self, y): current = self.lines[y].node while True: print(current) current = current.next_term if current is None: return # ========================== AST modification stuff ========================== # def insertTextNoSim(self, text): self.viewport_y = 0 self.tm.import_file(text) return def getTL(self): return self.getWindow().tl def getPL(self): return self.getWindow().pl def getLRP(self): return self.getWindow().lrp def getWindow(self): return self.window() def getEditorTab(self): return self.parent().parent().parent() def getScrollArea(self): return self.parent().parent() def createSubgrammarMenu(self, menu, change=False): self.sublanguage = None tmp = None if change: # try and find lbox and set cursor to previous node before getting # lookahead list root = self.tm.cursor.node.get_root() lbox = root.get_magicterminal() if lbox: tmp = self.tm.cursor.node self.tm.cursor.node = lbox.prev_term # Create actions bf = QFont() bf.setBold(True) valid_langs = [] for l in languages: if "<%s>" % l in self.tm.getLookaheadList(): valid_langs.append(l) if tmp: # undo cursor change self.tm.cursor.node = tmp if len(valid_langs) > 0: for l in valid_langs: item = QAction(str(l), menu) item.setData(l) self._set_icon(item, l) item.setFont(bf) menu.addAction(item) menu.addSeparator() for l in languages: item = QAction(str(l), menu) item.setData(l) self._set_icon(item, l) menu.addAction(item) return menu def showSubgrammarMenu(self): menu = QtGui.QMenu("Language", self) self.createSubgrammarMenu(menu) x,y = self.cursor_to_coordinate() action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht)) if action: self.sublanguage = action.data().toPyObject() self.edit_rightnode = True def _set_icon(self, mitem, lang): if lang.base.lower() == "html": icon = QIcon.fromTheme("text-xhtml+xml") else: icon = QIcon.fromTheme("text-x-" + lang.base.lower()) if icon.isNull(): icon = QIcon.fromTheme("application-x-" + lang.base.lower()) if icon.isNull(): icon = QIcon.fromTheme("text-x-generic") mitem.setIcon(icon) def showCodeCompletionMenu(self, l): menu = QtGui.QMenu( self ) # Create actions toolbar = QtGui.QToolBar() for n in l: path = [] for p in n.path: if p and p.name: path.append(p.name) if n.vartype: vartype = n.vartype while vartype.children != []: try: vartype = vartype.children[0] except KeyError: vartype = vartype.children["name"] text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind) elif n.kind == "method": text = self.cc_method(n) + " - %s" % (".".join(path)) else: text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind) item = toolbar.addAction(text, self.createCCFunc(n.name)) item.setIcon(QIcon("gui/" + n.kind + ".png")) menu.addAction(item) x,y = self.cursor_to_coordinate() menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht)) def cc_method(self, n): s = [n.name, "("] param_ln = n.astnode.children["params"] if isinstance(param_ln, ListNode): for p in param_ln.children: tmp = p.children["type"] if isinstance(tmp, AstNode): s.append(tmp.children["name"].symbol.name) else: s.append(tmp.symbol.name) s.append(" ") s.append(p.children["name"].symbol.name) if p != param_ln.children[-1]: s.append(", ") s.append(")") return "".join(s) def createMenuFunction(self, l): def action(): self.sublanguage = l self.edit_rightnode = True return action def createCCFunc(self, text): def action(): self.tm.pasteCompletion(text) return action def selectSubgrammar(self, item): pass def saveToJson(self, filename, swap=False): whitespaces = self.tm.get_mainparser().whitespaces root = self.tm.parsers[0][0].previous_version.parent language = self.tm.parsers[0][2] manager = JsonManager() manager.save(root, language, whitespaces, filename) if not swap: self.tm.changed = False self.emit(SIGNAL("painted()")) def loadFromJson(self, filename): manager = JsonManager() language_boxes = manager.load(filename) self.tm = TreeManager() self.tm.load_file(language_boxes) self.reset() def export(self, run=False, profile=False, source=None, debug=False): return self.tm.export(None, run, profile, source=source, debug=debug)
class BootstrapParser(object): def __init__(self, lr_type=1, whitespaces=False): self.lr_type = lr_type self.whitespaces = whitespaces # load (old) parser for grammar grammar self.rules = {} self.lrules = [] self.start_symbol = None self.incparser = None self.inclexer = None self.terminals = set() self.extra_alternatives = {} self.change_startrule = None self.options = {} self.precedences = [] def implicit_ws(self): if self.options.has_key("implicit_ws"): if self.options["implicit_ws"] == "true": return True return False def indentation_based(self): if self.options.has_key("indentation"): if self.options["indentation"] == "true": return True return False def parse(self, ecogrammar): # this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco)) from grammars.eco_grammar import eco_grammar as grammar self.lexer = IncrementalLexer(grammar.priorities) self.parser = IncParser(grammar.grammar, 1, True) self.parser.init_ast() self.ast = self.parser.previous_version.parent self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, grammar.name) self.treemanager.import_file(ecogrammar) if self.parser.last_status == False: raise Exception("Invalid input grammar: at %s %s" % (self.parser.error_node.prev_term, self.parser.error_node)) self.read_options() self.create_parser() self.create_lexer() def read_options(self): startrule = self.ast.children[1] # startrule assert startrule.symbol.name == "Startrule" grammar = startrule.children[1] assert grammar.symbol.name == "grammar" for element in grammar.children: if element.symbol.name == "options": break if element.symbol.name != "options": # grammar has no options print("warning: grammar has no options") # backwards compatibility if self.whitespaces: self.options["implicit_ws"] = "true" return options = element assert options.symbol.name == "options" self.parse_options(options) def parse_options(self, options): if options.children == []: return if len(options.children) > 0: assert options.children[0].symbol.name == "settings" self.parse_settings(options.children[0]) if len(options.children) > 1: assert options.children[1].symbol.name == "precedences" self.parse_precedences(options.children[1]) def parse_settings(self, options): if options.children == []: return if len(options.children) == 2: more = options.children[0] self.parse_settings(more) option = options.children[1] else: option = options.children[0] name = option.children[2].symbol.name choice = option.children[6] assert choice.symbol.name == "choice" self.options[name] = choice.children[0].symbol.name def parse_precedences(self, precedences): if precedences.children == []: return # recursively parse other precedences if len(precedences.children) == 2: more = precedences.children[0] self.parse_precedences(more) precedence = precedences.children[1] else: precedence = precedences.children[0] # parse single precedence name = precedence.children[0].symbol.name terminals = self.parse_precedence_symbols(precedence.children[2]) self.precedences.append((name, terminals)) def parse_precedence_symbols(self, symbol): s = [] for c in symbol.children: if c.symbol.name == "WS": continue if c.symbol.name == "terminals": rec_s = self.parse_precedence_symbols(symbol.children[0]) s.extend(rec_s) if c.lookup == "terminal": s.append(c.symbol.name[1:-1]) return s def create_parser(self, pickle_id = None): startrule = self.ast.children[1] # startrule grammar = startrule.children[1] parser = grammar.children[0] assert parser.symbol.name == "parser" self.parse_rules(parser) if self.implicit_ws(): ws_rule = Rule() ws_rule.symbol = Nonterminal("WS") ws_rule.add_alternative([Terminal("<ws>"), Nonterminal("WS")]) ws_rule.add_alternative([Terminal("<return>"), Nonterminal("WS")]) ws_rule.add_alternative([Terminal("<backslash>"), Terminal("<return>"), Nonterminal("WS")]) ws_rule.add_alternative([]) # or empty self.rules[ws_rule.symbol] = ws_rule # allow whitespace/comments at beginning of file start_rule = Rule() start_rule.symbol = Nonterminal("Startrule") start_rule.add_alternative([Nonterminal("WS"), self.start_symbol]) self.rules[start_rule.symbol] = start_rule self.start_symbol = start_rule.symbol incparser = IncParser() incparser.from_dict(self.rules, self.start_symbol, self.lr_type, self.implicit_ws(), pickle_id, self.precedences) incparser.init_ast() self.incparser = incparser def parse_rules(self, node): if node.children[0].symbol.name == "parser": self.parse_rules(node.children[0]) self.parse_rule(node.children[3]) elif node.children[0].symbol.name == "rule": self.parse_rule(node.children[0]) def parse_rule(self, node): name = node.children[0].symbol.name alternatives = self.parse_alternatives(node.children[4]) symbol = Nonterminal(name) if self.start_symbol is None: self.start_symbol = symbol if self.change_startrule and symbol.name == self.change_startrule: self.start_symbol = symbol r = Rule(symbol) for a in alternatives: r.add_alternative(a[0], a[1], a[2]) # add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes) if self.extra_alternatives.has_key(symbol.name): for n in self.extra_alternatives[symbol.name]: r.add_alternative([MagicTerminal(n), Nonterminal("WS")], None) self.rules[symbol] = r def parse_alternatives(self, node): if node.children[0].symbol.name == "alternatives": alternatives = self.parse_alternatives(node.children[0]) alternative = self.parse_alternative(node.children[3]) alternatives.append(alternative) return alternatives elif node.children[0].symbol.name == "right": return [self.parse_alternative(node.children[0])] def parse_alternative(self, node): if len(node.children) > 0: annotation = None prec = None for c in node.children: if c.symbol.name == "symbols": symbols = self.parse_symbols(c) if c.symbol.name == "prec": prec = self.parse_prec(c) if c.symbol.name == "annotations": annotation = self.parse_annotation(c) return (symbols, annotation, prec) else: return ([], None, None) def parse_prec(self, node): if node.children: c = node.children[2] return c.symbol.name[1:-1] def parse_symbols(self, node): if node.children[0].symbol.name == "symbols": symbols = self.parse_symbols(node.children[0]) symbol = self.parse_symbol(node.children[1]) symbols.append(symbol) if (isinstance(symbol, Terminal) or isinstance(symbol, MagicTerminal)) and self.implicit_ws(): symbols.append(Nonterminal("WS")) return symbols elif node.children[0].symbol.name == "symbol": l = [] symbol = self.parse_symbol(node.children[0]) l.append(symbol) if isinstance(symbol, Terminal) and self.implicit_ws(): l.append(Nonterminal("WS")) return l def parse_symbol(self, node): node = node.children[0] if node.lookup == "nonterminal": return Nonterminal(node.symbol.name) elif node.lookup == "terminal": self.terminals.add(node.symbol.name[1:-1]) return Terminal(node.symbol.name[1:-1]) elif node.lookup == "languagebox": return MagicTerminal(node.symbol.name) def parse_annotation(self, node): a_options = node.children[2] assert a_options.symbol.name == "a_options" if a_options.children[0].symbol.name == "astnode": return self.parse_astnode(a_options.children[0]) elif a_options.children[0].symbol.name == "expression": return self.parse_expression(a_options.children[0]) elif a_options.children[0].symbol.name == "forloop": return self.parse_foreach(a_options.children[0]) def parse_astnode(self, node): name = node.children[0].symbol.name children = self.parse_astnode_children(node.children[4]) d = {} for n, expr in children: d[n] = expr return AstNode(name, d) def parse_astnode_children(self, node): assert node.symbol.name == "astnode_children" if node.children[0].symbol.name == "astnode_child": return [self.parse_astnode_child(node.children[0])] elif node.children[0].symbol.name == "astnode_children": children = self.parse_astnode_children(node.children[0]) child = self.parse_astnode_child(node.children[3]) children.append(child) return children def parse_astnode_child(self, node): assert node.symbol.name == "astnode_child" name = node.children[0].symbol.name if node.children[4].symbol.name == "expression": expr = self.parse_expression(node.children[4]) elif node.children[4].symbol.name == "reference": expr = self.parse_reference(node.children[4]) return (name, expr) def parse_expression(self, node): if node.children[0].symbol.name == "node": return self.parse_node(node.children[0]) elif node.children[0].symbol.name == "list": return self.parse_list(node.children[0]) elif node.children[0].symbol.name == "node_ref": return self.parse_noderef(node.children[0]) else: expr1 = self.parse_expression(node.children[0]) if node.children[3].symbol.name == "node": expr2 = self.parse_node(node.children[3]) else: expr2 = self.parse_list(node.children[3]) return AddExpr(expr1, expr2) def parse_foreach(self, node): item = self.parse_node(node.children[4]) expr = self.parse_astnode(node.children[7]) return Foreach(node.symbol.name, item, expr) def parse_noderef(self, node): lookup = self.parse_node(node.children[0]) attr = node.children[3] lookup.attribute = attr.symbol.name return lookup def parse_node(self, node): return LookupExpr(int(node.children[2].symbol.name)) def parse_list(self, node): return ListExpr(self.parse_listloop(node.children[2])) def parse_reference(self, node): base = node.children[0].symbol.name ref = node.children[4].symbol.name return ReferenceExpr(base, ref) def parse_listloop(self, node): if len(node.children) == 0: return [] if node.children[0].symbol.name == "list_loop": l = self.parse_listloop(node.children[0]) element = self.parse_unknown(node.children[3]) l.append(element) return l else: return [self.parse_unknown(node.children[0])] def parse_unknown(self, node): if node.symbol.name == "node": return self.parse_node(node) elif node.symbol.name == "astnode": return self.parse_astnode(node) def create_lexer(self): startrule = self.ast.children[1] # startrule grammar = startrule.children[1] for element in grammar.children: if element.symbol.name == "lexer": break lexer = element assert lexer.symbol.name == "lexer" self.parse_lexer(lexer) names = [] regexs = [] for name, regex in self.lrules: names.append(name) regexs.append(regex) # add so far undefined terminals undefined_terminals = self.terminals.difference(set(names)) import re for t in undefined_terminals: names.insert(0, t) regexs.insert(0,re.escape(t)) self.inclexer = IncrementalLexerCF() self.inclexer.from_name_and_regex(names, regexs) if self.indentation_based(): self.inclexer.indentation_based = True def parse_lexer(self, lexer): if lexer.children[0].symbol.name == "lrule": self.parse_lrule(lexer.children[0]) elif lexer.children[0].symbol.name == "lexer": self.parse_lexer(lexer.children[0]) self.parse_lrule(lexer.children[1]) def parse_lrule(self, lrule): assert lrule.children[0].symbol.name == "tokenname" name = lrule.children[0].children[0].symbol.name regex = lrule.children[3].symbol.name[1:-1] self.lrules.append((name, regex))
class NodeEditor(QFrame): # ========================== init stuff ========================== # def __init__(self, parent=None): QtGui.QWidget.__init__(self, parent) self.infofont = QtGui.QFont('Courier', 6) self.infofontht = QtGui.QFontMetrics(self.infofont).height() + 3 self.infofontwt = QtGui.QFontMetrics(self.infofont).width(" ") self.viewport_y = 0 # top visible line self.imagemode = False self.image = None self.scroll_height = 0 self.scroll_width = 0 self.timer = QTimer(self) self.backuptimer = QTimer(self) self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer) self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer) self.backuptimer.start(30000) self.undotimer = QTimer(self) self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer) self.blinktimer = QTimer(self) self.blinktimer.start(500) self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer) self.show_cursor = True self.lightcolors = [QColor("#333333"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")] self.darkcolors = [QColor("#999999"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")] self.setCursor(Qt.IBeamCursor) def analysis_timer(self): if self.getWindow().show_namebinding(): self.tm.analyse() self.update() self.timer.stop() # save swap filename = self.getEditorTab().filename if filename: self.saveToJson(filename + ".swp", True) def backup_timer(self): filename = self.getEditorTab().filename if filename: self.saveToJson(filename + ".bak", True) def trigger_blinktimer(self): if self.timer.isActive(): self.show_cursor = True return self.show_cursor ^= True self.update() def trigger_undotimer(self): self.tm.save_current_version() self.undotimer.stop() def setImageMode(self, boolean): self.imagemode = boolean def reset(self): #self.getWindow().ui.scrollArea.horizontalScrollBar().setValue(0) #self.getWindow().ui.scrollArea.verticalScrollBar().setValue(0) self.update() def set_mainlanguage(self, parser, lexer, lang_name): self.tm = TreeManager() self.tm.add_parser(parser, lexer, lang_name) def set_sublanguage(self, language): self.sublanguage = language def event(self, event): if event.type() == QEvent.ToolTip: if QToolTip.isVisible(): QToolTip.hideText() event.ignore() return True pos = event.pos() temp_cursor = self.tm.cursor.copy() result = self.coordinate_to_cursor(pos.x(), pos.y()) node = self.tm.cursor.node self.tm.cursor.line = temp_cursor.line self.tm.cursor.node = temp_cursor.node self.tm.cursor.pos = temp_cursor.pos if not result: event.ignore() return True msg = self.tm.get_error(node) if not msg: msg = self.tm.get_error(node) if msg: QToolTip.showText(event.globalPos(), msg); return True return QFrame.event(self, event) # ========================== GUI related stuff ========================== # def blink(self): if self.show_cursor: self.show_cursor = 0 else: self.show_cursor = 1 self.update() def sliderChanged(self, value): change = self.viewport_y - value self.viewport_y = value self.update() def sliderXChanged(self, value): self.move(-value*self.fontwt,0) self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height()) if self.x() == 0: self.updateGeometry() self.update() def getScrollSizes(self): total_lines = 0 max_width = 0 for l in self.lines: total_lines += l.height max_width = max(max_width, l.width) max_visible_lines = self.geometry().height() / self.fontht self.scroll_height = max(0, total_lines - max_visible_lines) current_width = self.parentWidget().geometry().width() / self.fontwt self.scroll_width = max(0, max_width - current_width) def paintEvent(self, event): gfont = QApplication.instance().gfont self.font = gfont.font self.fontwt = gfont.fontwt self.fontht = gfont.fontht QtGui.QFrame.paintEvent(self, event) paint = QtGui.QPainter() if self.imagemode: self.image = QImage() paint.begin(self.image) else: paint.begin(self) paint.setFont(self.font) y = 0 x = 0 self.longest_column = 0 # calculate how many lines we need to show self.init_height = self.geometry().height() self.paintLines(paint, self.viewport_y) paint.end() total_lines = 0 max_width = 0 for l in self.lines: total_lines += l.height max_width = max(max_width, l.width) max_visible_lines = self.geometry().height() / self.fontht self.scroll_height = max(0, total_lines - max_visible_lines) current_width = self.parentWidget().geometry().width() / self.fontwt self.scroll_width = max(0, max_width - current_width) self.emit(SIGNAL("painted()")) # paint lines using new line manager def paintLines(self, paint, startline): # find internal line corresponding to visual line visual_line = 0 internal_line = 0 for l in self.tm.lines: if visual_line + l.height > startline: break visual_line += l.height internal_line += 1 x = 0 y = visual_line - startline # start drawing outside of viewport to display partial images self.paint_start = (internal_line, y) max_y = self.geometry().height() / self.fontht line = internal_line node = self.tm.lines[line].node self.paint_nodes(paint, node, x, y, line, max_y) #XXX if starting node is inside language box, init lbox with amout of languge boxes def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0): settings = QSettings("softdev", "Eco") if settings.value("app_theme", "Light").toString() == "Dark": alpha = 100 colors = self.darkcolors else: alpha = 40 colors = self.lightcolors first_node = node selected_language = self.tm.mainroot error_node = self.tm.get_mainparser().error_node error_node = self.fix_errornode(error_node) highlighter = self.get_highlighter(node) selection_start = min(self.tm.selection_start, self.tm.selection_end) selection_end = max(self.tm.selection_start, self.tm.selection_end) draw_selection_start = (0,0,0) draw_selection_end = (0,0,0) start_lbox = self.get_languagebox(node) editor = self.get_editor(node) self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node) #XXX get initial x for langbox if start_lbox: lbox += 1 if start_lbox and self.selected_lbox is start_lbox: draw_lbox = True else: draw_lbox = False draw_all_boxes = self.getWindow().show_languageboxes() self.lines = self.tm.lines self.cursor = self.tm.cursor self.lines[line].height = 1 # reset height draw_cursor = True #l_x = [0] while y < max_y: # if we found a language box, continue drawing inside of it if isinstance(node.symbol, MagicTerminal): #l_x.append(x) #node.pos = x lbox += 1 lbnode = node.symbol.ast if self.selected_lbox is node: draw_lbox = True selected_language = lbnode else: draw_lbox = False node = lbnode.children[0] highlighter = self.get_highlighter(node) editor = self.get_editor(node) error_node = self.tm.get_parser(lbnode).error_node error_node = self.fix_errornode(error_node) continue if isinstance(node, EOS): lbnode = self.get_languagebox(node) if self.cursor.node is lbnode: self.draw_cursor(paint, x, 5 + y * self.fontht) if lbnode: #l_x.pop() if lbox > 0: lbox -= 1 node = lbnode.next_term highlighter = self.get_highlighter(node) editor = self.get_editor(node) if self.selected_lbox is lbnode: draw_lbox = False lbnode = self.get_languagebox(node) if lbnode and self.selected_lbox is lbnode: draw_lbox = True error_node = self.tm.get_parser(lbnode.symbol.ast).error_node error_node = self.fix_errornode(error_node) else: error_node = self.tm.get_mainparser().error_node error_node = self.fix_errornode(error_node) continue else: self.lines[line].width = x / self.fontwt break # draw language boxes if lbox > 0 and (draw_lbox or draw_all_boxes): #color = self.nesting_colors[lbox % 5] if draw_all_boxes: color = colors[(lbox-1) % len(colors)] color.setAlpha(alpha) else: color = colors[0] color.setAlpha(alpha) if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box color = colors[-1] # overwrite color color.setAlpha(alpha) editor.update_image(node) if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal): if not node.image or node.plain_mode: paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color) # prepare selection drawing if node is selection_start.node: if node.lookup == "<return>": sel_x = x else: sel_x = x + selection_start.pos * self.fontwt draw_selection_start = (sel_x, y, line) if node is selection_end.node: draw_selection_end = (x + selection_end.pos * self.fontwt, y, line) # draw node dx, dy = editor.paint_node(paint, node, x, y, highlighter) x += dx #y += dy self.lines[line].height = max(self.lines[line].height, dy) # after we drew a return, update line information if node.lookup == "<return>" and not node is first_node: # draw lbox to end of line if draw_lbox or (draw_all_boxes and lbox > 0): paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color) self.lines[line].width = x / self.fontwt x = 0#l_x[-1] y += self.lines[line].height line += 1 self.lines[line].height = 1 # reset height # draw cursor if node is self.cursor.node and self.show_cursor: draw_x = max(0, x-dx) cursor_pos = self.cursor.pos if node.symbol.name == "\r": cursor_pos = 0 if node.image and not node.plain_mode: draw_x = x cursor_pos = 0 self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 5 + y * self.fontht) if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor: draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3) paint.drawRect(draw_cursor_at) # set lbox info coordinates infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht) draw_cursor = False # draw squiggly line if node is error_node or (self.getWindow().show_namebinding() and self.tm.has_error(node)): if isinstance(node, EOS): length = self.fontwt else: length = len(node.symbol.name)*self.fontwt if isinstance(node.symbol, MagicTerminal): self.draw_vertical_squiggly_line(paint,x,y) else: if self.tm.has_error(node): color = "orange" else: color = "red" self.draw_squiggly_line(paint,x-length,y,length, color) node = node.next_term if selection_start != selection_end: self.draw_selection(paint, draw_selection_start, draw_selection_end) # paint infobox if False: lang_name = self.parser_langs[selected_language] lang_status = self.parsers[selected_language].last_status if lang_status is True: color = QColor(100,255,100) else: color = QColor(255,100,100) paint.setFont(self.infofont) paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*self.infofontwt, self.infofontht), color) paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name) paint.setFont(self.font) return x, y, line def fix_errornode(self, error_node): if not error_node: return while isinstance(error_node.symbol, IndentationTerminal): error_node = error_node.prev_term return error_node def draw_cursor(self, paint, x, y): pen = paint.pen() colorhex = self.palette().color(QPalette.Text) pen.setColor(QColor(colorhex)) paint.setPen(pen) draw_cursor_at = QRect(x, y, 0, self.fontht - 3) paint.drawRect(draw_cursor_at) def draw_vertical_squiggly_line(self, paint, x, y): paint.setPen(Qt.CustomDashLine) pen = paint.pen() pen.setDashPattern([2,2]) pen.setColor(QColor("red")) paint.setPen(pen) y = 3+y*self.fontht paint.drawLine(x-1, y, x-1, y+self.fontht) paint.drawLine(x, y+2, x, y+self.fontht) paint.setPen(Qt.SolidLine) def draw_squiggly_line(self, paint, x, y, length, color): paint.setPen(Qt.CustomDashLine) pen = paint.pen() pen.setDashPattern([2,2]) pen.setColor(QColor(color)) paint.setPen(pen) #x -= length y = (y+1)*self.fontht+1 paint.drawLine(x, y, x+length, y) paint.drawLine(x+2, y+1, x+2+length, y+1) paint.setPen(Qt.SolidLine) def draw_selection(self, paint, draw_selection_start, draw_selection_end): x1, y1, line1 = draw_selection_start x2, y2, line2 = draw_selection_end if y1 == y2: paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100)) else: paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, self.tm.lines[line1].width*self.fontwt - x1, self.fontht), QColor(0,0,255,100)) y = y1 + self.tm.lines[line1].height for i in range(line1+1, line2): paint.fillRect(QRectF(0, 3 + y * self.fontht, self.tm.lines[i].width*self.fontwt, self.fontht), QColor(0,0,255,100)) y = y + self.tm.lines[i].height paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100)) def get_highlighter(self, node): root = node.get_root() base = lang_dict[self.tm.get_language(root)].base s = syntaxhighlighter.get_highlighter(base, self.palette()) return s def get_languagebox(self, node): root = node.get_root() lbox = root.get_magicterminal() return lbox def get_editor(self, node): root = node.get_root() base = lang_dict[self.tm.get_language(root)].base return editor.get_editor(base, self.fontwt, self.fontht) def focusNextPrevChild(self, b): # don't switch to next widget on TAB return False def mousePressEvent(self, e): if e.button() == Qt.LeftButton: self.tm.input_log.append("# mousePressEvent") self.coordinate_to_cursor(e.x(), e.y()) # self.tm.cursor = cursor self.tm.selection_start = self.tm.cursor.copy() self.tm.selection_end = self.tm.cursor.copy() self.tm.input_log.append("self.selection_start = self.cursor.copy()") self.tm.input_log.append("self.selection_end = self.cursor.copy()") #self.tm.fix_cursor_on_image() self.getWindow().showLookahead() self.update() def mouseDoubleClickEvent(self, e): if e.button() == Qt.LeftButton: self.coordinate_to_cursor(e.x(), e.y()) node = self.tm.get_node_from_cursor() lbox = self.get_languagebox(node) if lbox and lbox.symbol.name == "<IPython>": if lbox.plain_mode is False: lbox.plain_mode = True else: lbox.plain_mode = False self.update() return elif node.image is None: self.tm.selection_start = self.tm.cursor.copy() self.tm.selection_start.node = self.tm.cursor.find_previous_visible(self.tm.cursor.node) self.tm.selection_start.pos = len(self.tm.selection_start.node.symbol.name) self.tm.selection_end = self.tm.cursor.copy() self.tm.selection_end.pos = len(self.tm.selection_end.node.symbol.name) self.tm.cursor.pos = self.tm.selection_end.pos self.tm.cursor.node = self.tm.selection_end.node self.update() return if node.plain_mode is False: node.plain_mode = True self.tm.cursor.pos = len(node.symbol.name) else: node.plain_mode = False self.update() def cursor_to_coordinate(self): y = 0 for l in self.tm.lines[:self.cursor.line]: y += l.height * self.fontht x = self.tm.cursor.get_x() * self.fontwt y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht return (x,y) def coordinate_to_cursor(self, x, y): mouse_y = y / self.fontht first_line = self.paint_start[0] y_offset = self.paint_start[1] y = y_offset line = first_line while line < len(self.tm.lines) - 1: y += self.tm.lines[line].height if y > mouse_y: break line += 1 self.tm.cursor.line = line cursor_x = int(round(float(x) / self.fontwt)) self.tm.cursor.move_to_x(cursor_x, self.tm.lines) self.tm.input_log.append("self.cursor.line = %s" % str(line)) self.tm.log_input("cursor.move_to_x", str(cursor_x), "self.lines") if mouse_y > y or self.tm.cursor.get_x() != cursor_x: return False return True def mouseMoveEvent(self, e): # apparaently this is only called when a mouse button is clicked while # the mouse is moving if self.tm.input_log[-2].startswith("self.cursor.move_to_x"): # only log the last move event self.tm.input_log.pop() self.tm.input_log.pop() self.tm.input_log.pop() self.coordinate_to_cursor(e.x(), e.y()) self.tm.selection_end = self.tm.cursor.copy() self.tm.input_log.append("self.selection_end = self.cursor.copy()") self.update() def key_to_string(self, key): if key == Qt.Key_Up: return "up" if key == Qt.Key_Down: return "down" if key == Qt.Key_Left: return "left" if key == Qt.Key_Right: return "right" def keyPressEvent(self, e): startundotimer = False self.timer.start(500) self.show_cursor = True if e.key() in [Qt.Key_Shift, Qt.Key_Alt, Qt.Key_Control, Qt.Key_Meta, Qt.Key_AltGr]: if e.key() == Qt.Key_Shift: self.tm.key_shift() return text = e.text() self.edit_rightnode = False # has been processes in get_nodes_at_pos -> reset if e.key() == Qt.Key_Escape: self.tm.key_escape() elif e.key() == Qt.Key_Backspace: startundotimer = True self.tm.key_backspace() elif e.key() in [Qt.Key_Up, Qt.Key_Down, Qt.Key_Left, Qt.Key_Right]: if e.modifiers() == Qt.ShiftModifier: self.tm.key_cursors(self.key_to_string(e.key()), True) elif e.modifiers() == Qt.ControlModifier: self.tm.ctrl_cursor(self.key_to_string(e.key())) else: self.tm.key_cursors(self.key_to_string(e.key()), False) elif e.key() == Qt.Key_Home: self.tm.key_home(e.modifiers() == Qt.ShiftModifier) elif e.key() == Qt.Key_End: self.tm.key_end(e.modifiers() == Qt.ShiftModifier) elif e.key() == Qt.Key_Delete: startundotimer = True self.tm.key_delete() elif e.key() == Qt.Key_F3: self.tm.find_next() elif e.key() in [Qt.Key_PageUp, Qt.Key_PageDown]: pass # ignore those keys elif e.modifiers() in [Qt.ControlModifier, Qt.AltModifier]: # User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't # sensibly insert into the text. pass else: startundotimer = True if e.key() == Qt.Key_Tab: text = " " else: text = e.text() if text.toUtf8() not in whitelist: logging.debug("Key %s not supported" % text) return self.tm.key_normal(text) self.getWindow().btReparse([]) self.update() self.emit(SIGNAL("keypress(QKeyEvent)"), e) self.getWindow().showLookahead() if startundotimer: self.undotimer.start(500) def showLanguageBoxMenu(self): self.showSubgrammarMenu() self.create_languagebox() def create_languagebox(self): if self.sublanguage: if self.tm.hasSelection(): self.tm.surround_with_languagebox(self.sublanguage) else: self.tm.add_languagebox(self.sublanguage) def change_languagebox(self): if self.sublanguage: self.tm.change_languagebox(self.sublanguage) def showCodeCompletion(self): l = self.tm.getCompletion() if l: self.showCodeCompletionMenu(l) def println(self, prestring, y): node = self.lines[y].node.next_term x = [] while node is not None and node.symbol.name != "\r": x.append(node.symbol.name) node = node.next_term print(prestring, "".join(x)) def print_line(self, y): current = self.lines[y].node while True: print(current) current = current.next_term if current is None: return # ========================== AST modification stuff ========================== # def insertTextNoSim(self, text): self.viewport_y = 0 self.tm.import_file(text) return def getTL(self): return self.getWindow().tl def getPL(self): return self.getWindow().pl def getLRP(self): return self.getWindow().lrp def getWindow(self): return self.window() def getEditorTab(self): return self.parent().parent().parent() def getScrollArea(self): return self.parent().parent() def createSubgrammarMenu(self, menu, change=False): self.sublanguage = None tmp = None if change: # try and find lbox and set cursor to previous node before getting # lookahead list root = self.tm.cursor.node.get_root() lbox = root.get_magicterminal() if lbox: tmp = self.tm.cursor.node self.tm.cursor.node = lbox.prev_term # Create actions bf = QFont() bf.setBold(True) valid_langs = [] for l in languages: if "<%s>" % l in self.tm.getLookaheadList(): valid_langs.append(l) if tmp: # undo cursor change self.tm.cursor.node = tmp if len(valid_langs) > 0: for l in valid_langs: item = QAction(str(l), menu) item.setData(l) self._set_icon(item, l) item.setFont(bf) menu.addAction(item) menu.addSeparator() for l in languages: item = QAction(str(l), menu) item.setData(l) self._set_icon(item, l) menu.addAction(item) return menu def showSubgrammarMenu(self): menu = QtGui.QMenu("Language", self) self.createSubgrammarMenu(menu) x,y = self.cursor_to_coordinate() action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht)) if action: self.sublanguage = action.data().toPyObject() self.edit_rightnode = True def _set_icon(self, mitem, lang): if lang.base.lower() == "html": icon = QIcon.fromTheme("text-xhtml+xml") else: icon = QIcon.fromTheme("text-x-" + lang.base.lower()) if icon.isNull(): icon = QIcon.fromTheme("application-x-" + lang.base.lower()) if icon.isNull(): icon = QIcon.fromTheme("text-x-generic") mitem.setIcon(icon) def showCodeCompletionMenu(self, l): menu = QtGui.QMenu( self ) # Create actions toolbar = QtGui.QToolBar() for n in l: path = [] for p in n.path: if p and p.name: path.append(p.name) if n.vartype: vartype = n.vartype while vartype.children != []: try: vartype = vartype.children[0] except KeyError: vartype = vartype.children["name"] text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind) elif n.kind == "method": text = self.cc_method(n) + " - %s" % (".".join(path)) else: text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind) item = toolbar.addAction(text, self.createCCFunc(n.name)) item.setIcon(QIcon("gui/" + n.kind + ".png")) menu.addAction(item) x,y = self.cursor_to_coordinate() menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht)) def cc_method(self, n): s = [n.name, "("] param_ln = n.astnode.children["params"] if isinstance(param_ln, ListNode): for p in param_ln.children: tmp = p.children["type"] if isinstance(tmp, AstNode): s.append(tmp.children["name"].symbol.name) else: s.append(tmp.symbol.name) s.append(" ") s.append(p.children["name"].symbol.name) if p != param_ln.children[-1]: s.append(", ") s.append(")") return "".join(s) def createMenuFunction(self, l): def action(): self.sublanguage = l self.edit_rightnode = True return action def createCCFunc(self, text): def action(): self.tm.pasteCompletion(text) return action def selectSubgrammar(self, item): pass def saveToJson(self, filename, swap=False): whitespaces = self.tm.get_mainparser().whitespaces root = self.tm.parsers[0][0].previous_version.parent language = self.tm.parsers[0][2] manager = JsonManager() manager.save(root, language, whitespaces, filename) if not swap: self.tm.changed = False self.emit(SIGNAL("painted()")) def loadFromJson(self, filename): manager = JsonManager() language_boxes = manager.load(filename) self.tm = TreeManager() self.tm.load_file(language_boxes) self.reset() def export(self, run=False): return self.tm.export(None, run)
def random_deletion(self): """Delete random characters within a program.""" print "Running random_deletion on {}".format(self.filename) program = self.program self.treemanager.import_file(program) assert self.parser.last_status == True self.text_compare(program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) random_lines = random_lines[: 20] # restrict to 20 lines to reduce runtime start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.treemanager.cursor_reset() self.log.append("self.treemanager.cursor_reset()") self.move(DOWN, linenr) self.log.append("self.move(DOWN, %s)" % linenr) self.move(RIGHT, col) self.log.append("self.move(RIGHT, %s)" % col) self.log.append("self.treemanager.key_delete()") x = self.treemanager.key_delete() if x == "eos": continue self.treemanager.undo_snapshot() self.log.append("self.treemanager.undo_snapshot()") end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(program) t1 = TreeManager() parser, lexer = self.lang.load() parser.init_ast() t1.add_parser(parser, lexer, self.lang.name) t1.set_font_test(7, 17) t1.import_file(self.program) assert self.parser.last_status == True assert parser.last_status == True self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent)
def random_insertdelete(self): """Insert and delete random characters at random locations within a program.""" print "Running random_insertdelete on {}".format(self.filename) self.reset() self.treemanager.import_file(self.program) assert self.parser.last_status == True self.text_compare(self.program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) random_lines = random_lines[: 20] # restrict to 20 lines to reduce runtime start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.log.append("self.treemanager.cursor_reset()") self.log.append("self.move(%s, %s)" % (DOWN, linenr)) self.log.append("self.move(%s, %s)" % (RIGHT, col)) self.treemanager.cursor_reset() self.move(DOWN, linenr) self.move(RIGHT, col) k = self.get_random_key() if k in [ "a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7" ]: # for a few characters DELETE instead of INSERT self.log.append("self.treemanager.key_delete()") x = self.treemanager.key_delete() else: rk = self.get_random_key() self.log.append("self.treemanager.key_normal(%s)" % rk) x = self.treemanager.key_normal(rk) if x == "eos": continue self.log.append("self.treemanager.undo_snapshot()") self.treemanager.undo_snapshot() end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) t1 = TreeManager() parser, lexer = self.lang.load() t1.add_parser(parser, lexer, self.lang.name) t1.import_file(self.program) self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent)
class FuzzyTester(): """Runs test that randomly modify a given program and tests for exceptions. If an exception occurs, or the modifications had unwanted side-effects, a log is saved which can be used to create a stand-alone test.""" def __init__(self, filename): _, ext = os.path.splitext(filename) self.setlang(ext_to_lang[ext]) with open(filename) as f: self.program = f.read() self.log = [] self.filename = filename def reset(self): self.parser.reset() self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, self.lang.name) def setlang(self, lang): self.lang = lang_dict[lang] self.parser, self.lexer = lang_dict[lang].load() self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, self.lang.name) def move(self, direction, times): for i in range(times): self.treemanager.cursor_movement(direction) def text_compare(self, original): """Compare the textual representation of two programs. Faster than tree_compare, but less accurate.""" original = original.replace("\r", "").split("\n") current = self.treemanager.export_as_text("/dev/null").replace( "\r", "").split("\n") for i in xrange(len(current)): assert original[i] == current[i] def next_node(self, node): if node.children: return node.children[0] while node.right_sibling() is None: node = node.parent return node.right_sibling() def tree_compare(self, node1, node2): """Given two root nodes, compares that both trees are equivalent""" while True: assert node1.symbol == node2.symbol if node1.right: assert node1.right.symbol == node2.right.symbol if node1.next_term: assert node1.next_term.symbol == node2.next_term.symbol if isinstance(node1.symbol, MagicTerminal): self.tree_compare(node1.symbol.ast, node2.symbol.ast) if isinstance(node1, EOS) and isinstance(node2, EOS): break node1 = self.next_node(node1) node2 = self.next_node(node2) def get_random_key(self): keys = list( "abcdefghijklmnopqrstuvwxyz0123456789 \r:,.[]{}()!$%^&*()_+=") return random.choice(keys) def random_deletion(self): """Delete random characters within a program.""" print "Running random_deletion on {}".format(self.filename) program = self.program self.treemanager.import_file(program) assert self.parser.last_status == True self.text_compare(program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) random_lines = random_lines[: 20] # restrict to 20 lines to reduce runtime start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.treemanager.cursor_reset() self.log.append("self.treemanager.cursor_reset()") self.move(DOWN, linenr) self.log.append("self.move(DOWN, %s)" % linenr) self.move(RIGHT, col) self.log.append("self.move(RIGHT, %s)" % col) self.log.append("self.treemanager.key_delete()") x = self.treemanager.key_delete() if x == "eos": continue self.treemanager.undo_snapshot() self.log.append("self.treemanager.undo_snapshot()") end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(program) t1 = TreeManager() parser, lexer = self.lang.load() parser.init_ast() t1.add_parser(parser, lexer, self.lang.name) t1.set_font_test(7, 17) t1.import_file(self.program) assert self.parser.last_status == True assert parser.last_status == True self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent) def random_insertion(self): """Insert random characters at random locations within a program.""" print "Running random_insert on {}".format(self.filename) self.reset() self.treemanager.import_file(self.program) assert self.parser.last_status == True self.text_compare(self.program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.log.append("self.treemanager.cursor_reset()") self.log.append("self.move(DOWN, %s)" % linenr) self.log.append("self.move(RIGHT, %s)" % col) self.treemanager.cursor_reset() self.move(DOWN, linenr) self.move(RIGHT, col) k = self.get_random_key() self.log.append("self.treemanager.key_normal(%s)" % repr(k)) x = self.treemanager.key_normal(k) if x == "eos": continue self.log.append("self.treemanager.undo_snapshot()") self.treemanager.undo_snapshot() end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) t1 = TreeManager() parser, lexer = self.lang.load() t1.add_parser(parser, lexer, self.lang.name) t1.import_file(self.program) self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent) def random_insertdelete(self): """Insert and delete random characters at random locations within a program.""" print "Running random_insertdelete on {}".format(self.filename) self.reset() self.treemanager.import_file(self.program) assert self.parser.last_status == True self.text_compare(self.program) line_count = len(self.treemanager.lines) random_lines = range(line_count) random.shuffle(random_lines) random_lines = random_lines[: 20] # restrict to 20 lines to reduce runtime start_version = self.treemanager.version for linenr in random_lines: cols = range(20) random.shuffle(cols) for col in cols: self.log.append("self.treemanager.cursor_reset()") self.log.append("self.move(%s, %s)" % (DOWN, linenr)) self.log.append("self.move(%s, %s)" % (RIGHT, col)) self.treemanager.cursor_reset() self.move(DOWN, linenr) self.move(RIGHT, col) k = self.get_random_key() if k in [ "a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7" ]: # for a few characters DELETE instead of INSERT self.log.append("self.treemanager.key_delete()") x = self.treemanager.key_delete() else: rk = self.get_random_key() self.log.append("self.treemanager.key_normal(%s)" % rk) x = self.treemanager.key_normal(rk) if x == "eos": continue self.log.append("self.treemanager.undo_snapshot()") self.treemanager.undo_snapshot() end_version = self.treemanager.version broken = self.treemanager.export_as_text() # undo all and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) # redo all and compare with broken while self.treemanager.version < end_version: self.treemanager.key_shift_ctrl_z() self.text_compare(broken) # undo again and compare with original while self.treemanager.version > start_version: self.treemanager.key_ctrl_z() self.text_compare(self.program) t1 = TreeManager() parser, lexer = self.lang.load() t1.add_parser(parser, lexer, self.lang.name) t1.import_file(self.program) self.tree_compare(self.parser.previous_version.parent, parser.previous_version.parent) def run(self): try: ft.random_deletion() self.reset() ft.random_insertion() self.reset() ft.random_insertdelete() except Exception as e: traceback.print_exc() print "Written log to 'fuzzy.log'." with open("fuzzy.log", "w") as f: for l in self.log: f.write(l) f.write("\n") else: print "Passed."
class FuzzyLboxStats: def __init__(self, main, sub): parser, lexer = main.load() self.lexer = lexer self.parser = parser self.ast = parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(parser, lexer, main.name) parser.setup_autolbox(main.name) self.sub = sub self.inserted = 0 def load_main(self, filename): f = open(filename, "r") content = f.read() f.close() self.treemanager.import_file(content) self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) def load_expr(self, filename): f = open(filename, "r") content = f.read() f.close() self.replexprs = self.find_expressions(content, self.sub_repl_str) def set_replace(self, main, sub): self.main_repl_str = main self.sub_repl_str = sub def find_nonterms_by_name(self, tm, name): l = [] bos = tm.get_bos() eos = tm.get_eos() node = bos.right_sibling() while node is not eos: if node.symbol.name == name: l.append(node) if node.children: node = node.children[0] continue node = next_node(node) return l def find_expressions(self, program, expr): parser, lexer = self.sub.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, self.sub.name) treemanager.import_file(program) # find all sub expressions l = self.find_nonterms_by_name(treemanager, expr) return [subtree_to_text(st).rstrip() for st in l] def insert_python_expression(self, expr): for c in expr: self.treemanager.key_normal(c) def delete_expr(self, expr): # find first term and last term # select + delete node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[0] else: node = next_node(node) first = node node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[-1] else: node = prev_node(node) last = node if first.deleted or last.deleted: return None self.treemanager.select_nodes(first, last) deleted = self.treemanager.copySelection() self.treemanager.deleteSelection() return deleted def run(self): assert len(self.treemanager.parsers) == 1 print self.main_repl_str, len( [subtree_to_text(x) for x in self.mainexprs]) print self.sub_repl_str, len(self.replexprs) random.shuffle(self.mainexprs) for e in self.mainexprs[:10]: if e.get_root() is None: continue deleted = self.delete_expr(e) before = len(self.treemanager.parsers) if deleted: choice = random.choice(self.replexprs) print " Replacing '{}' with '{}':".format( truncate(deleted), truncate(choice)) self.insert_python_expression(choice) valid = self.parser.last_status if before == len(self.treemanager.parsers): result = "No box inserted" else: result = "Box inserted" self.inserted += 1 print " => {} ({})".format(result, valid) else: print "Replacing '{}' with '{}':\n => Already deleted".format( truncate(subtree_to_text(e)), truncate(choice)) print("Boxes inserted: {}/{}".format(self.inserted, 10))