def random_insertion(self):
"""Insert random characters at random locations within a program."""
print "Running random_insert on {}".format(self.filename)
self.reset()
self.treemanager.import_file(self.program)
assert self.parser.last_status == True
self.text_compare(self.program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.log.append("self.treemanager.cursor_reset()")
self.log.append("self.move(DOWN, %s)" % linenr)
self.log.append("self.move(RIGHT, %s)" % col)
self.treemanager.cursor_reset()
self.move(DOWN, linenr)
self.move(RIGHT, col)
k = self.get_random_key()
self.log.append("self.treemanager.key_normal(%s)" % repr(k))
x = self.treemanager.key_normal(k)
if x == "eos":
continue
self.log.append("self.treemanager.undo_snapshot()")
self.treemanager.undo_snapshot()
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
t1 = TreeManager()
parser, lexer = self.lang.load()
t1.add_parser(parser, lexer, self.lang.name)
t1.import_file(self.program)
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
def find_expressions(self, program, expr):
parser, lexer = self.sub.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, self.sub.name)
treemanager.import_file(program)
# find all sub expressions
l = self.find_nonterms_by_name(treemanager, expr)
return [subtree_to_text(st).rstrip() for st in l]
def find_subtree(grammar, program, cond):
parser, lexer = grammar.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, grammar.name)
try:
treemanager.import_file(program)
except Exception:
return None
if parser.last_status is False:
return None
# find all sub expressions
l = find_nonterms_by_name(treemanager, cond)
return [subtree_to_text(st).rstrip() for st in l]
def run(filename):
# Load grammar
parser, lexer = java.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, "")
# Load file
print("Running", filename)
with open(filename, "r") as f:
treemanager.import_file(f.read())
if not parser.last_status:
print("Couldn't parse file. Skip.")
return
filesize = os.path.getsize(filename)
timings = []
# Insert `1+` after every `=`
node = treemanager.get_bos()
while type(node) is not EOS:
if node.symbol.name == "=":
node.insert("1+", len(node.symbol.name))
node.mark_changed()
treemanager.relex(node)
treemanager.post_keypress("")
treemanager.save_current_version()
parser.prev_version = treemanager.version
parser.reference_version = treemanager.reference_version
start = time()
parser.inc_parse()
end = time()
parser.top_down_reuse()
treemanager.save_current_version(
postparse=True) # save post parse tree
if parser.last_status:
treemanager.reference_version = treemanager.version
TreeManager.version = treemanager.version
if not parser.last_status:
break
timings.append(str(end - start))
node = node.next_term
if len(timings) > 0:
with open("results.csv", "a+") as f:
f.write("{} {} {}".format(filename, filesize, ",".join(timings)))
f.write("\n")
class FuzzyMinimiser(object):
def __init__(self, lang, program, test):
grm = lang_dict[language]
parser, lexer = grm.load()
parser.init_ast()
self.parser = parser
self.lexer = lexer
self.grm = grm
self.treemanager = TreeManager()
self.treemanager.add_parser(parser, lexer, grm.name)
self.treemanager.version = 1
self.treemanager.last_saved_version = 1
self.program = program
self.test = test
self.pos = 0
self.lastexception = None
def reset(self):
self.parser.reset()
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, self.grm)
def auto(self):
logging.info("Running auto-mode")
# try to reduce snapshots first
while True:
self.original = self.test[:] # remember original tests
self.reduce("snapshot")
if self.test == self.original:
logging.info("No more snapshot reductions possible.")
break
# try single reduce next
while True:
self.original = self.test[:] # remember original tests
self.reduce("single")
if self.test == self.original:
logging.info("No more single reductions possible.")
break
def reduce(self, mode):
self.pos = 0
logging.info("Pos: {} Len: {} Start {}-reduce".format(self.pos, len(self.test), mode))
try:
# Run without changes to store exception
self.run()
except Exception as e:
self.lastexception = e
while self.pos < len(self.test):
self.delete(mode)
try:
self.run() # check if first run causes error
logging.info("Pos: {} Len: {} Fixed".format(self.pos, len(self.test)))
self.revert()
except Exception as e:
if not self.same_exception(e):
logging.info("Pos: {} Len: {} Wrong exception: '{}'".format(self.pos, len(self.test), e))
self.revert()
continue
else:
logging.info("Pos: {} Len: {} Still failing: '{}'".format(self.pos, len(self.test), e))
self.save() # save progress
self.save()
def same_exception(self, e):
if not self.lastexception:
self.lastexception = e
return True
if type(self.lastexception) is type(e) and \
self.lastexception.args == e.args:
return True
return False
def delete(self, mode):
self.old = self.test[:] # copy
if mode == "single":
del self.test[self.pos:self.pos+4] # remove one keystroke
if self.pos < len(self.test) and \
self.test[self.pos] == "self.treemanager.undo_snapshot()":
self.test.pop(self.pos)
elif mode == "snapshot":
# removes 1 snapshot consisting of 20 keystrokes (each 4 lines) and
# a `self.treemanager.undo_snapshot()` line at the end
del self.test[self.pos:self.pos+81]
def revert(self):
diff = len(self.old) - len(self.test)
self.test = self.old[:] # revert changes
self.pos += diff # and skip test lines that are needed
def set_destination(self, dest):
filename = dest + ".1"
i = 1
while os.path.isfile(filename):
filename = dest + ".{}".format(i)
i += 1
self.dest = filename
logging.info("Setting destination to '{}'".format(filename))
def save(self):
f = open(self.dest, "w")
for l in self.test:
f.write(l)
f.write("\n")
f.close()
def run(self):
self.reset()
self.treemanager.import_file(self.program)
for l in self.test:
exec(l)
def move(self, direction, times):
for i in range(times): self.treemanager.cursor_movement(direction)
class FuzzyLboxStats:
def __init__(self, main, sub):
parser, lexer = main.load()
self.lexer = lexer
self.parser = parser
self.ast = parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(parser, lexer, main.name)
self.treemanager.option_autolbox_insert = True
self.langname = main.name
parser.setup_autolbox(main.name, lexer)
self.sub = sub
self.inserted = 0
self.log = []
def load_main(self, filename):
self.filename = filename
f = open(filename, "r")
self.content = f.read()
f.close()
self.content.replace("\n", "\r")
self.treemanager.import_file(self.content)
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
self.minver = self.treemanager.version
def reset(self):
self.parser.reset()
self.ast = self.parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, self.langname)
self.treemanager.import_file(self.content)
self.treemanager.option_autolbox_insert = True
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
def load_expr(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.replexprs = self.find_expressions(content, self.sub_repl_str)
def load_expr_from_json(self, filename):
import json
with open(filename) as f:
self.replexprs = json.load(f)
def set_replace(self, main, sub):
self.main_repl_str = main
self.sub_repl_str = sub
def find_nonterms_by_name(self, tm, name):
l = []
bos = tm.get_bos()
eos = tm.get_eos()
node = bos.right_sibling()
while node is not eos:
if validnonterm(node, name):
l.append(node)
node = next_node(node)
continue
if node.children:
node = node.children[0]
else:
node = next_node(node)
return l
def find_expressions(self, program, expr):
parser, lexer = self.sub.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, self.sub.name)
treemanager.import_file(program)
# find all sub expressions
l = self.find_nonterms_by_name(treemanager, expr)
return [subtree_to_text(st).rstrip() for st in l]
def insert_python_expression(self, expr):
for c in expr:
self.treemanager.key_normal(c)
def delete_expr(self, expr):
# find first term and last term
# select + delete
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[0]
else:
node = next_node(node)
first = node
if isinstance(first, MultiTextNode):
first = first.children[0]
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[-1]
else:
node = prev_node(node)
while node.lookup == "<ws>" or node.lookup == "<return>":
node = node.prev_term
last = node
if isinstance(last, MultiTextNode):
last = last.children[-1]
if first.deleted or last.deleted:
return None
self.treemanager.select_nodes(first, last)
deleted = self.treemanager.copySelection()
self.treemanager.deleteSelection()
return deleted
def multi_len(self, autos):
r = []
for start, end, _, _ in autos:
l = 0
while start is not end:
l += len(start.symbol.name)
start = start.next_term
r.append(l)
return r
def run(self, main_samples=None, sub_samples=None):
assert len(self.treemanager.parsers) == 1
ops = self.main_repl_str, len(
[subtree_to_text(x) for x in self.mainexprs])
preversion = self.treemanager.version
inserted_error = 0
inserted_valid = 0
noinsert_error = 0
noinsert_valid = 0
noinsert_multi = 0
# pick random exprs from main
if not main_samples:
samplesize = 10
if len(self.mainexprs) < 10:
samplesize = len(self.mainexprs)
sample = random.sample(range(len(self.mainexprs)),
samplesize) # store this for repeatability
exprchoices = [self.mainexprs[i] for i in sample]
self.main_samples = sample
else:
self.main_samples = main_samples
exprchoices = [self.mainexprs[i] for i in main_samples]
if not sub_samples:
# pick random exprs from sub
sample = random.sample(range(len(self.replexprs)),
len(exprchoices))
replchoices = [self.replexprs[i] for i in sample]
self.sub_samples = sample
else:
self.sub_samples = sub_samples
replchoices = [self.replexprs[i] for i in sub_samples]
for i in range(len(exprchoices)):
e = exprchoices[i]
if e.get_root() is None:
continue
before = len(self.treemanager.parsers)
deleted = self.delete_expr(e)
mboxes = None
ilength = None
if deleted:
choice = replchoices[i].strip()
choice_len = len(choice)
if debug:
print " Replacing '{}' with '{}':".format(
repr(truncate(deleted)), repr(choice))
start = time.time()
cursor = self.treemanager.cursor
if cursor.node.lookup != "<ws>":
# Insert a leading space in situations like `return(x)`.
choice = " " + choice
if cursor.node.symbol.name[
cursor.
pos:] == "" and cursor.node.next_term.lookup != "<ws>":
# Insert trailing space if there is none
choice = choice + " "
self.insert_python_expression(choice)
itime = time.time() - start
valid = self.parser.last_status
if before == len(self.treemanager.parsers):
if len(
self.parser.error_nodes
) > 0 and self.parser.error_nodes[0].autobox and len(
self.parser.error_nodes[0].autobox) > 1:
noinsert_multi += 1
result = "No box inserted (Multi)"
outcome = "multi"
mboxes = self.multi_len(
self.parser.error_nodes[0].autobox)
elif valid:
noinsert_valid += 1
result = "No box inserted (Valid)"
outcome = "valid"
else:
noinsert_error += 1
result = "No box inserted (Error)"
outcome = "error"
else:
result = "Box inserted"
innervalid = self.treemanager.parsers[-1][0].last_status
self.inserted += 1
recent_box = self.treemanager.parsers[-1][
0].previous_version.parent
lbox_len = self.lbox_length(recent_box)
ilength = (lbox_len, choice_len)
if valid and innervalid:
inserted_valid += 1
outcome = "ok"
else:
inserted_error += 1
outcome = "inerr"
if debug: print " => {} ({})".format(result, valid)
nlboxes = len(self.treemanager.parsers)
self.log.append([
self.filename,
repr(deleted),
repr(choice), outcome, itime, mboxes, ilength, nlboxes
])
else:
if debug:
print "Replacing '{}' with '{}':\n => Already deleted".format(
truncate(subtree_to_text(e)), truncate(choice))
self.undo(self.minver)
exprchoices = [self.mainexprs[i] for i in self.main_samples]
if debug:
print("Boxes inserted: {}/{}".format(self.inserted, ops))
print("Valid insertions:", inserted_valid)
print("Invalid insertions:", inserted_error)
print("No insertion (valid):", noinsert_valid)
print("No insertion (error):", noinsert_error)
print("No insertion (multi):", noinsert_multi)
return (inserted_valid, inserted_error, noinsert_valid, noinsert_error,
noinsert_multi)
def undo(self, version):
# reset everything
self.reset()
return
while self.treemanager.version != version:
before = self.treemanager.version
self.treemanager.version -= 1
self.treemanager.recover_version("undo",
self.treemanager.version + 1)
self.treemanager.cursor.load(self.treemanager.version,
self.treemanager.lines)
if before == self.treemanager.version:
exit("Error")
def lbox_length(self, root):
l = []
node = root.children[0]
eos = root.children[-1]
while node is not eos:
if not node.deleted:
l.append(node.symbol.name)
node = node.next_term
s = "".join(l).strip()
return len(s)
class BootstrapParser(object):
def __init__(self, lr_type=1, whitespaces=False):
self.lr_type = lr_type
self.whitespaces = whitespaces
# load (old) parser for grammar grammar
self.rules = {}
self.lrules = []
self.start_symbol = None
self.incparser = None
self.inclexer = None
self.terminals = set()
self.extra_alternatives = {}
self.change_startrule = None
self.options = {"nowhitespace": []}
self.precedences = []
self.current_rulename = ""
self.all_terminals = set()
self.functions = []
self.prod_ids = {}
def implicit_ws(self):
if self.options.has_key("implicit_ws"):
if self.options["implicit_ws"] == "true":
return True
return False
def implicit_newlines(self):
if self.options.has_key("implicit_newlines"):
if self.options["implicit_newlines"] == "false":
return False
return True
def indentation_based(self):
if self.options.has_key("indentation"):
if self.options["indentation"] == "true":
return True
return False
def parse(self, ecogrammar):
# this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco))
from grammars.eco_grammar import eco_grammar as grammar
self.lexer = IncrementalLexer(grammar.priorities)
self.parser = IncParser(grammar.grammar, 1, True)
self.parser.init_ast()
self.ast = self.parser.previous_version.parent
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, grammar.name)
self.treemanager.import_file(ecogrammar)
if self.parser.last_status == False:
raise Exception("Invalid input grammar due to syntax errors")
self.read_options()
self.parse_both()
self.create_parser()
self.create_lexer()
def parse_both(self):
# parse rules
startrule = self.ast.children[1] # startrule
grammar = startrule.children[1]
parser = grammar.children[0]
assert parser.symbol.name == "parser"
self.parse_rules(parser)
# parse lexer
startrule = self.ast.children[1] # startrule
grammar = startrule.children[1]
for element in grammar.children:
if element.symbol.name == "lexer":
break
lexer = element
assert lexer.symbol.name == "lexer"
self.parse_lexer(lexer)
for name, regex in self.lrules:
# collect terminals for parser modifications
self.all_terminals.add(name)
def read_options(self):
startrule = self.ast.children[1] # startrule
assert startrule.symbol.name == "Startrule"
grammar = startrule.children[1]
assert grammar.symbol.name == "grammar"
for element in grammar.children:
if element.symbol.name == "options":
break
if element.symbol.name != "options":
# grammar has no options
print("warning: grammar has no options")
# backwards compatibility
if self.whitespaces:
self.options["implicit_ws"] = "true"
return
options = element
assert options.symbol.name == "options"
self.parse_options(options)
def parse_options(self, options):
if options.children == []:
return
if len(options.children) > 0:
assert options.children[0].symbol.name == "settings"
self.parse_settings(options.children[0])
if len(options.children) > 1:
assert options.children[1].symbol.name == "precedences"
self.parse_precedences(options.children[1])
def parse_settings(self, options):
if options.children == []:
return
if len(options.children) == 2:
more = options.children[0]
self.parse_settings(more)
option = options.children[1]
else:
option = options.children[0]
name = option.children[2].symbol.name
choice = option.children[6]
assert choice.symbol.name == "choice"
if choice.children[0].symbol.name == "choice_list":
self.options[name] = self.parse_choicelist(choice.children[0])
else:
self.options[name] = choice.children[0].symbol.name
def parse_choicelist(self, symbol):
s = []
for c in symbol.children:
if c.symbol.name == ",":
continue
if c.symbol.name == "WS":
continue
if c.lookup == "nonterminal":
s.append(c.symbol.name)
continue
if c.symbol.name == "choice_list":
rec_s = self.parse_choicelist(symbol.children[0])
s.extend(rec_s)
continue
return s
def parse_precedences(self, precedences):
if precedences.children == []:
return
# recursively parse other precedences
if len(precedences.children) == 2:
more = precedences.children[0]
self.parse_precedences(more)
precedence = precedences.children[1]
else:
precedence = precedences.children[0]
# parse single precedence
name = precedence.children[0].symbol.name
terminals = self.parse_precedence_symbols(precedence.children[2])
self.precedences.append((name, terminals))
def parse_precedence_symbols(self, symbol):
s = []
for c in symbol.children:
if c.symbol.name == "WS":
continue
if c.symbol.name == "terminals":
rec_s = self.parse_precedence_symbols(symbol.children[0])
s.extend(rec_s)
if c.lookup == "terminal":
s.append(c.symbol.name[1:-1])
return s
def create_parser(self, pickle_id=None):
self.all_terminals.update(self.terminals)
for fname, terminals, parentrule in self.functions:
if fname.startswith("*match_until"):
if Nonterminal(fname) not in self.rules:
r = Rule(Nonterminal(fname))
for t in self.all_terminals:
if t not in terminals:
r.add_alternative(
[Nonterminal(fname),
Terminal(t)], None, t)
r.add_alternative([])
self.rules[r.symbol] = r
# remove whitespace before special rule from parent rule, e.g.
# multistring ::= "MLS" WS *match_until "MLS" WS
# ^ this WS causes shift/reduce conflicts
prule = self.rules[Nonterminal(parentrule)]
for a in prule.alternatives:
for i in range(len(a)):
sym = a[i]
if sym.name == "WS":
if len(a) > i + 1 and a[i + 1].name.startswith(
"*match_until"):
a.pop(i)
break
if self.implicit_ws():
ws_rule = Rule()
ws_rule.symbol = Nonterminal("WS")
ws_rule.add_alternative([Nonterminal("WS"), Terminal("<ws>")])
# get comment rule
if self.options.has_key('comment_rule'):
cmt_rules = self.options['comment_rule']
for cmt_rule in cmt_rules:
if Nonterminal(cmt_rule) in self.rules:
ws_rule.add_alternative(
[Nonterminal("WS"),
Nonterminal("comment")])
if self.implicit_newlines():
ws_rule.add_alternative(
[Nonterminal("WS"),
Terminal("<return>")])
ws_rule.add_alternative([
Nonterminal("WS"),
Terminal("<backslash>"),
Terminal("<return>")
])
ws_rule.add_alternative([]) # or empty
self.rules[ws_rule.symbol] = ws_rule
for a in ws_rule.alternatives:
self.prod_ids[Production(ws_rule.symbol,
a)] = len(self.prod_ids)
# allow whitespace/comments at beginning of file
start_rule = Rule()
start_rule.symbol = Nonterminal("Startrule")
start_rule.add_alternative([Nonterminal("WS"), self.start_symbol])
self.rules[start_rule.symbol] = start_rule
self.prod_ids[Production(start_rule.symbol,
start_rule.alternatives[0])] = len(
self.prod_ids)
self.start_symbol = start_rule.symbol
incparser = IncParser()
incparser.from_dict(self.rules, self.start_symbol, self.lr_type,
self.implicit_ws(), pickle_id, self.precedences,
self.prod_ids)
incparser.init_ast()
self.incparser = incparser
def parse_rules(self, node):
if node.children[0].symbol.name == "parser":
self.parse_rules(node.children[0])
self.parse_rule(node.children[3])
elif node.children[0].symbol.name == "rule":
self.parse_rule(node.children[0])
def parse_rule(self, node):
name = node.children[0].symbol.name
self.current_rulename = name
alternatives = self.parse_alternatives(node.children[4])
symbol = Nonterminal(name)
if self.start_symbol is None:
self.start_symbol = symbol
if self.change_startrule and symbol.name == self.change_startrule:
self.start_symbol = symbol
r = Rule(symbol)
for a in alternatives:
r.add_alternative(a[0], a[1], a[2])
self.prod_ids[Production(symbol, a[0])] = len(self.prod_ids)
# add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes)
if self.extra_alternatives.has_key(symbol.name):
for n in self.extra_alternatives[symbol.name]:
a = [MagicTerminal(n), Nonterminal("WS")]
r.add_alternative(a)
self.prod_ids[Production(symbol, a)] = len(self.prod_ids)
self.rules[symbol] = r
def parse_alternatives(self, node):
if node.children[0].symbol.name == "alternatives":
alternatives = self.parse_alternatives(node.children[0])
alternative = self.parse_alternative(node.children[3])
alternatives.append(alternative)
return alternatives
elif node.children[0].symbol.name == "right":
return [self.parse_alternative(node.children[0])]
def parse_alternative(self, node):
if len(node.children) > 0:
annotation = None
prec = None
for c in node.children:
if c.symbol.name == "symbols":
symbols = self.parse_symbols(c)
if c.symbol.name == "prec":
prec = self.parse_prec(c)
if c.symbol.name == "annotations":
annotation = self.parse_annotation(c)
return (symbols, annotation, prec)
else:
return ([], None, None)
def parse_prec(self, node):
if node.children:
c = node.children[2]
return c.symbol.name[1:-1]
def parse_symbols(self, node):
if node.children[0].symbol.name == "symbols":
symbols = self.parse_symbols(node.children[0])
symbol = self.parse_symbol(node.children[1])
symbols.append(symbol)
if (
isinstance(symbol, Terminal)
or isinstance(symbol, MagicTerminal)
) and self.implicit_ws(
) and self.current_rulename not in self.options["nowhitespace"]:
symbols.append(Nonterminal("WS"))
return symbols
elif node.children[0].symbol.name == "symbol":
l = []
symbol = self.parse_symbol(node.children[0])
l.append(symbol)
if isinstance(symbol, Terminal) and self.implicit_ws(
) and self.current_rulename not in self.options["nowhitespace"]:
l.append(Nonterminal("WS"))
return l
def parse_symbol(self, node):
node = node.children[0]
if node.lookup == "nonterminal":
return Nonterminal(node.symbol.name)
elif node.lookup == "terminal":
if node.symbol.name != "\"<eos>\"":
self.terminals.add(node.symbol.name[1:-1])
return Terminal(node.symbol.name[1:-1])
elif node.lookup == "languagebox":
return MagicTerminal(node.symbol.name)
elif node.symbol.name == "function":
return self.parse_function(node)
def parse_function(self, node):
fname = node.children[0].symbol.name
terminals = self.parse_fargs(node.children[4])
safe_name = "*%s%s" % (fname, hash(frozenset(terminals)))
self.functions.append((safe_name, terminals, self.current_rulename))
return Nonterminal(safe_name)
def parse_fargs(self, symbol):
s = []
for c in symbol.children:
if c.symbol.name == ",":
continue
if c.symbol.name == "WS":
continue
if c.lookup == "terminal":
s.append(c.symbol.name[1:-1])
continue
if c.symbol.name == "f_args":
rec_s = self.parse_fargs(symbol.children[0])
s.extend(rec_s)
return s
def parse_annotation(self, node):
a_options = node.children[2]
assert a_options.symbol.name == "a_options"
if a_options.children[0].symbol.name == "astnode":
return self.parse_astnode(a_options.children[0])
elif a_options.children[0].symbol.name == "expression":
return self.parse_expression(a_options.children[0])
elif a_options.children[0].symbol.name == "forloop":
return self.parse_foreach(a_options.children[0])
def parse_astnode(self, node):
name = node.children[0].symbol.name
children = self.parse_astnode_children(node.children[4])
d = {}
for n, expr in children:
d[n] = expr
return AstNode(name, d)
def parse_astnode_children(self, node):
assert node.symbol.name == "astnode_children"
if node.children[0].symbol.name == "astnode_child":
return [self.parse_astnode_child(node.children[0])]
elif node.children[0].symbol.name == "astnode_children":
children = self.parse_astnode_children(node.children[0])
child = self.parse_astnode_child(node.children[3])
children.append(child)
return children
def parse_astnode_child(self, node):
assert node.symbol.name == "astnode_child"
name = node.children[0].symbol.name
if node.children[4].symbol.name == "expression":
expr = self.parse_expression(node.children[4])
elif node.children[4].symbol.name == "reference":
expr = self.parse_reference(node.children[4])
return (name, expr)
def parse_expression(self, node):
if node.children[0].symbol.name == "node":
return self.parse_node(node.children[0])
elif node.children[0].symbol.name == "list":
return self.parse_list(node.children[0])
elif node.children[0].symbol.name == "node_ref":
return self.parse_noderef(node.children[0])
else:
expr1 = self.parse_expression(node.children[0])
if node.children[3].symbol.name == "node":
expr2 = self.parse_node(node.children[3])
else:
expr2 = self.parse_list(node.children[3])
return AddExpr(expr1, expr2)
def parse_foreach(self, node):
item = self.parse_node(node.children[4])
expr = self.parse_astnode(node.children[7])
return Foreach(node.symbol.name, item, expr)
def parse_noderef(self, node):
lookup = self.parse_node(node.children[0])
attr = node.children[3]
lookup.attribute = attr.symbol.name
return lookup
def parse_node(self, node):
return LookupExpr(int(node.children[2].symbol.name))
def parse_list(self, node):
return ListExpr(self.parse_listloop(node.children[2]))
def parse_reference(self, node):
base = node.children[0].symbol.name
ref = node.children[4].symbol.name
return ReferenceExpr(base, ref)
def parse_listloop(self, node):
if len(node.children) == 0:
return []
if node.children[0].symbol.name == "list_loop":
l = self.parse_listloop(node.children[0])
element = self.parse_unknown(node.children[3])
l.append(element)
return l
else:
return [self.parse_unknown(node.children[0])]
def parse_unknown(self, node):
if node.symbol.name == "node":
return self.parse_node(node)
elif node.symbol.name == "astnode":
return self.parse_astnode(node)
def create_lexer(self, buildlexer=True):
names = []
regexs = []
for name, regex in self.lrules:
names.append(name)
self.all_terminals.add(name)
regexs.append(regex)
# add so far undefined terminals
undefined_terminals = self.terminals.difference(set(names))
import re
for t in undefined_terminals:
names.insert(0, t)
regexs.insert(0, re.escape(t))
if not buildlexer:
self.inclexer = (names, regexs)
return
self.inclexer = IncrementalLexerCF()
self.inclexer.from_name_and_regex(names, regexs)
if self.indentation_based():
self.inclexer.indentation_based = True
def parse_lexer(self, lexer):
if lexer.children[0].symbol.name == "lrule":
self.parse_lrule(lexer.children[0])
elif lexer.children[0].symbol.name == "lexer":
self.parse_lexer(lexer.children[0])
self.parse_lrule(lexer.children[1])
def parse_lrule(self, lrule):
assert lrule.children[0].symbol.name == "tokenname"
name = lrule.children[0].children[0].symbol.name
regex = lrule.children[3].symbol.name[1:-1]
self.lrules.append((name, regex))
class Test_MultiTextNodePython:
def setup_class(cls):
parser, lexer = python.load()
cls.lexer = lexer
cls.parser = parser
cls.parser.init_ast()
cls.ast = cls.parser.previous_version
cls.treemanager = TreeManager()
cls.treemanager.add_parser(cls.parser, cls.lexer, python.name)
cls.treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite
def reset(self):
self.parser.reset()
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, calc.name)
self.treemanager.set_font_test(7, 17)
def test_simple(self):
self.reset()
inputstring = "x = \"\"\"abcdef\"\"\""
for c in inputstring:
self.treemanager.key_normal(c)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_normal("\r")
def test_relex_over_indentation(self):
self.reset()
inputstring = """class X:
x = 1
def x():
pass
y = 2"""
self.treemanager.import_file(inputstring)
assert self.parser.last_status == True
self.treemanager.cursor_movement(DOWN)
self.treemanager.cursor_movement(DOWN)
self.treemanager.cursor_movement(DOWN)
self.treemanager.key_end()
assert self.treemanager.cursor.node.symbol.name == "pass"
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.cursor_movement(UP)
self.treemanager.cursor_movement(UP)
self.treemanager.key_end()
assert self.treemanager.cursor.node.symbol.name == "1"
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
assert self.parser.last_status == True
def test_indentation_to_string_and_back(self):
self.reset()
inputstring = """class X:
a
b"""
self.treemanager.import_file(inputstring)
assert self.parser.last_status == True
self.treemanager.cursor_movement(DOWN)
self.treemanager.cursor_movement(DOWN)
self.treemanager.key_end()
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.cursor_movement(UP)
self.treemanager.cursor_movement(UP)
self.treemanager.key_home()
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
self.treemanager.key_normal("\"")
assert self.parser.last_status == True
self.treemanager.cursor_movement(DOWN)
self.treemanager.cursor_movement(DOWN)
self.treemanager.key_end()
self.treemanager.key_backspace()
assert self.parser.last_status == False
def test_remember_open_lexing_states(self):
self.reset()
inputstring = """x = 1
y = 2"""
self.treemanager.import_file(inputstring)
assert self.parser.last_status == True
self.treemanager.key_end()
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_normal("\"")
#assert self.parser.last_status == False # unfinished lexing jobs
self.treemanager.key_end()
self.treemanager.key_normal("\"")
assert self.parser.last_status == True
def test_triplequote_string(self):
self.reset()
inputstring = 'x="""abc"""'
for i in inputstring:
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""abc"""'
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_normal("\"")
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '""'
assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"'
assert bos.next_term.next_term.next_term.next_term.lookback == 1
assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == 'c'
assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2
assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
assert bos.next_term.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"'
self.treemanager.key_normal("\"")
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '""'
assert bos.next_term.next_term.next_term.next_term.symbol.name == '"ab"'
assert bos.next_term.next_term.next_term.next_term.lookback == 1
assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '"c"'
assert bos.next_term.next_term.next_term.next_term.next_term.lookback == 2
assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
self.treemanager.key_normal("\"")
#assert self.parser.last_status == False
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""'
assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c'
assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
assert bos.next_term.next_term.next_term.next_term.next_term.next_term.symbol.name == '"'
self.treemanager.key_end()
self.treemanager.key_backspace()
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""ab"""'
assert bos.next_term.next_term.next_term.next_term.symbol.name == 'c'
assert bos.next_term.next_term.next_term.next_term.next_term.symbol.name == '""'
def test_ignore_nonlbox_x80(self):
self.reset()
inputstring = 'x="""ab\x80c"""'
for i in inputstring:
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""ab\x80c"""'
def test_multinode_from_the_start(self):
self.reset()
inputstring = '''x="""a\rbc"""'''
for i in inputstring:
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'
def test_multinode_and_nonlbox_x80(self):
self.reset()
inputstring = '''x="""a\x80bc"""'''
for i in inputstring:
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""'
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_normal("\r")
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'
def test_multinode_nonlbox_and_lbox(self):
self.reset()
inputstring = '''x="""a\x80bc"""'''
for i in inputstring:
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '"""a\x80bc"""'
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.add_languagebox(lang_dict["SQL"])
self.treemanager.key_normal("S")
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "x"
assert bos.next_term.next_term.symbol.name == "="
assert bos.next_term.next_term.next_term.symbol.name == '<Multinode>'
multi = bos.next_term.next_term.next_term
assert multi.children[0].symbol.name == "\"\"\"a\x80b"
assert type(multi.children[1].symbol) is MagicTerminal
assert multi.children[2].symbol.name == "c\"\"\""
def test_multinode_merged_first(self):
self.reset()
inputstring = '''"""a\rbc"""'''
for i in inputstring:
self.treemanager.key_normal(i)
for i in 'def"""':
self.treemanager.key_normal(i)
bos = self.parser.previous_version.parent.children[0]
assert bos.next_term.symbol.name == "<Multinode>"
assert bos.next_term.next_term.symbol.name == 'def'
assert bos.next_term.next_term.next_term.symbol.name == '""'
assert bos.next_term.next_term.next_term.next_term.symbol.name == '"'
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_backspace()
self.treemanager.key_backspace()
self.treemanager.key_backspace()
assert bos.next_term.symbol.name == "<Multinode>"
assert bos.next_term.next_term.symbol.name == "NEWLINE"
assert bos.next_term.next_term.next_term.symbol.name == "eos"
def test_multinode_string_bug(self):
self.reset()
inputstring = '''x="abc"'''
for i in inputstring:
self.treemanager.key_normal(i)
self.treemanager.cursor_movement(LEFT)
self.treemanager.add_languagebox(lang_dict["SQL"])
self.treemanager.key_normal("x")
bos = self.parser.previous_version.parent.children[0]
x = bos.next_term
assert x.symbol.name == "x"
eq = x.next_term
assert eq.symbol.name == "="
multi = eq.next_term
assert multi.lookup == "dstring"
assert multi.symbol.name == "<Multinode>"
self.treemanager.cursor_movement(RIGHT)
self.treemanager.key_backspace()
# removing the ending quote results in a lexingerror,
# so the multinode remains
assert eq.next_term.symbol.name == "<Multinode>"
# now remove the first quote, which should lead to the destruction of
# the multinode
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.cursor_movement(LEFT)
self.treemanager.key_backspace()
assert eq.next_term.symbol.name == "abc"
def test_multinode_string_bug2(self):
self.reset()
inputstring = '''x="abc"'''
for i in inputstring:
self.treemanager.key_normal(i)
self.treemanager.cursor_movement(LEFT)
self.treemanager.add_languagebox(lang_dict["SQL"])
self.treemanager.key_normal("x")
self.treemanager.leave_languagebox()
self.treemanager.key_normal("z")
bos = self.parser.previous_version.parent.children[0]
x = bos.next_term
assert x.symbol.name == "x"
eq = x.next_term
assert eq.symbol.name == "="
multi = eq.next_term
assert multi.children[0].symbol.name == "\"abc"
assert multi.children[1].symbol.name == "<SQL>"
assert multi.children[2].symbol.name == "z\""
whitespace = True
lexer = IncrementalLexer(grammar.priorities)
parser = IncParser(grammar.grammar, 1, whitespace)
parser.init_ast()
ast = parser.previous_version
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, grammar.name)
treemanager.set_font_test(7, 17) # hard coded. PyQt segfaults in test suite
inputstring = """import abc.xyz as efg
from x import z
class Test:
def x():
if x == 1:
z = 3 + 4 * 5
elif x == 2:
for x in range(2,10):
print x
else:
z = 4
return 1 if a==b else 2"""
treemanager.import_file(inputstring)
viewer = Viewer("pydot")
viewer.get_tree_image(treemanager.get_mainparser().previous_version.parent, [], whitespace, ast=True)
import os
os.system("xdg-open " + viewer.image)
class NodeEditor(QFrame):
# ========================== init stuff ========================== #
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent)
self.viewport_y = 0 # top visible line
self.imagemode = False
self.image = None
self.scroll_height = 0
self.scroll_width = 0
self.timer = QTimer(self)
self.backuptimer = QTimer(self)
self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer)
self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer)
self.backuptimer.start(30000)
self.undotimer = QTimer(self)
self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer)
self.blinktimer = QTimer(self)
self.blinktimer.start(500)
self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer)
self.show_cursor = True
self.boxcolors = [QColor("#DC322F"), QColor("#268BD2"), QColor("#2Ac598"), QColor("#D33682"), QColor("#B58900"), QColor("#859900")]
self.setCursor(Qt.IBeamCursor)
# Semi-transparent overlay.
# Used to display heat-map visualisation of profiler info, etc.
self.overlay = Overlay(self)
# Always show the overlay, users hide visualisations with the HUD.
self.overlay.show()
# Show / don't show HUD visualisations.
self.hud_callgraph = False
self.hud_eval = False
self.hud_heat_map = False
self.hud_types = False
def hud_show_callgraph(self):
self.hud_callgraph = True
self.hud_eval = False
self.hud_heat_map = False
self.hud_types = False
def hud_show_eval(self):
self.hud_callgraph = False
self.hud_eval = True
self.hud_heat_map = False
self.hud_types = False
def hud_show_types(self):
self.hud_callgraph = False
self.hud_eval = False
self.hud_heat_map = False
self.hud_types = True
def hud_show_heat_map(self):
self.hud_callgraph = False
self.hud_eval = False
self.hud_heat_map = True
self.hud_types = False
def hud_off(self):
self.hud_callgraph = False
self.hud_eval = False
self.hud_heat_map = False
self.hud_types = False
def focusOutEvent(self, event):
self.blinktimer.stop()
self.show_cursor = True
self.update()
def focusInEvent(self, event):
self.blinktimer.start()
def resizeEvent(self, event):
self.overlay.resize(event.size())
event.accept()
def analysis_timer(self):
if self.getWindow().show_namebinding():
self.tm.analyse()
self.update()
self.timer.stop()
# save swap
filename = self.getEditorTab().filename
if filename:
self.saveToJson(filename + ".swp", True)
def backup_timer(self):
filename = self.getEditorTab().filename
if filename:
self.saveToJson(filename + ".bak", True)
def trigger_blinktimer(self):
if self.timer.isActive():
self.show_cursor = True
return
self.show_cursor ^= True
self.update()
def trigger_undotimer(self):
self.tm.undo_snapshot()
self.undotimer.stop()
def setImageMode(self, boolean):
self.imagemode = boolean
def reset(self):
self.update()
def set_mainlanguage(self, parser, lexer, lang_name):
self.tm = TreeManager()
self.tm.add_parser(parser, lexer, lang_name)
def get_mainlanguage(self):
return self.tm.parsers[0][2]
def set_sublanguage(self, language):
self.sublanguage = language
def event(self, event):
if event.type() == QEvent.ToolTip:
pos = event.pos()
temp_cursor = self.tm.cursor.copy()
result = self.coordinate_to_cursor(pos.x(), pos.y())
node = self.tm.cursor.node
self.tm.cursor.line = temp_cursor.line
self.tm.cursor.node = temp_cursor.node
self.tm.cursor.pos = temp_cursor.pos
if not result:
QToolTip.hideText()
event.ignore()
return True
# Draw errors, if there are any.
msg = self.tm.get_error(node)
if msg:
QToolTip.showText(event.globalPos(), msg)
return True
# Draw tooltips if there are any. Tooltips can appear with any HUD
# visualisation.
elif (self.hud_callgraph or self.hud_eval or self.hud_types or
self.hud_heat_map):
strings = []
annotes = node.get_annotations_with_hint(ToolTip)
if self.hud_callgraph:
for annote in annotes:
if annote.has_hint(HUDCallgraph):
strings.append(annote.annotation)
elif self.hud_eval:
for annote in annotes:
if annote.has_hint(HUDEval):
strings.append(annote.annotation)
elif self.hud_types:
for annote in annotes:
if annote.has_hint(HUDTypes):
strings.append(annote.annotation)
elif self.hud_heat_map:
for annote in annotes:
if annote.has_hint(HUDHeatmap):
strings.append(annote.annotation)
msg = "\n".join(strings)
if msg.strip() != "":
if self.tm.tool_data_is_dirty:
msg += "\n[Warning: Information may be out of date.]"
QToolTip.showText(event.globalPos(), msg)
return True
QToolTip.hideText()
event.ignore()
return True
return QFrame.event(self, event)
# ========================== GUI related stuff ========================== #
def blink(self):
if self.show_cursor:
self.show_cursor = 0
else:
self.show_cursor = 1
self.update()
def sliderChanged(self, value):
self.viewport_y = value
self.overlay.start_line = self.viewport_y
self.update()
def sliderXChanged(self, value):
self.move(-value*self.fontwt,0)
self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height())
if self.x() == 0:
self.updateGeometry()
self.update()
def getScrollSizes(self):
total_lines = 0
max_width = 0
for l in self.lines:
total_lines += l.height
max_width = max(max_width, l.width)
max_visible_lines = self.geometry().height() / self.fontht
self.scroll_height = max(0, total_lines - max_visible_lines)
current_width = self.parentWidget().geometry().width() / self.fontwt
self.scroll_width = max(0, max_width - current_width)
def paintEvent(self, event):
# Clear data in the visualisation overlay
self.overlay.clear_data()
gfont = QApplication.instance().gfont
self.font = gfont.font
self.fontwt = gfont.fontwt
self.fontht = gfont.fontht
self.fontd = gfont.fontd
QtGui.QFrame.paintEvent(self, event)
paint = QtGui.QPainter()
if self.imagemode:
self.image = QImage()
paint.begin(self.image)
else:
paint.begin(self)
paint.setFont(self.font)
self.longest_column = 0
# calculate how many lines we need to show
self.init_height = self.geometry().height()
self.paintLines(paint, self.viewport_y)
paint.end()
total_lines = 0
max_width = 0
for l in self.lines:
total_lines += l.height
max_width = max(max_width, l.width)
max_visible_lines = self.geometry().height() / self.fontht
self.scroll_height = max(0, total_lines - max_visible_lines)
current_width = self.parentWidget().geometry().width() / self.fontwt
self.scroll_width = max(0, max_width - current_width)
railroad_annotations = self.tm.get_all_annotations_with_hint(Railroad)
self.overlay.add_railroad_data(railroad_annotations)
self.emit(SIGNAL("painted()"))
# paint lines using new line manager
def paintLines(self, paint, startline):
# find internal line corresponding to visual line
visual_line = 0
internal_line = 0
for l in self.tm.lines:
if visual_line + l.height > startline:
break
visual_line += l.height
internal_line += 1
x = 0
y = visual_line - startline # start drawing outside of viewport to display partial images
self.paint_start = (internal_line, y)
max_y = self.geometry().height() / self.fontht
line = internal_line
node = self.tm.lines[line].node
_, _, self.end_line = self.paint_nodes(paint, node, x, y, line, max_y)
#XXX if starting node is inside language box, init lbox with amount of language boxes
def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0):
settings = QSettings("softdev", "Eco")
colors = self.boxcolors
if settings.value("app_theme", "Light").toString() in ["Dark"]:
alpha = 100
self.highlight_line_color = QColor(250,250,250,20)
elif settings.value("app_theme", "Light").toString() in ["Gruvbox"]:
alpha = 100
self.highlight_line_color = QColor(250,250,250,20)
else:
alpha = 60
self.highlight_line_color = QColor(0,0,0,10)
self.show_highlight_line = settings.value("highlight_line", False).toBool()
first_node = node
selected_language = self.tm.mainroot
error_node = self.tm.get_mainparser().error_node
error_node = self.fix_errornode(error_node)
highlighter = self.get_highlighter(node)
selection_start = min(self.tm.selection_start, self.tm.selection_end)
selection_end = max(self.tm.selection_start, self.tm.selection_end)
draw_selection_start = (0,0,0)
draw_selection_end = (0,0,0)
start_lbox = self.get_languagebox(node)
renderer = self.get_renderer(node)
self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node)
#XXX get initial x for langbox
if start_lbox:
lbox += 1
if start_lbox and self.selected_lbox is start_lbox:
draw_lbox = True
else:
draw_lbox = False
draw_all_boxes = self.getWindow().show_languageboxes()
self.lines = self.tm.lines
self.cursor = self.tm.cursor
self.lines[line].height = 1 # reset height
draw_cursor = True
show_namebinding = self.getWindow().show_namebinding()
while y < max_y:
# if we found a language box, continue drawing inside of it
if isinstance(node.symbol, MagicTerminal):
lbox += 1
lbnode = node.symbol.ast
if self.selected_lbox is node:
color = colors[(lbox-1) % len(colors)]
self.draw_lbox_bracket(paint, '[', node, x, y, color)
draw_lbox = True
selected_language = lbnode
else:
draw_lbox = False
node = lbnode.children[0]
highlighter = self.get_highlighter(node)
renderer = self.get_renderer(node)
error_node = self.tm.get_parser(lbnode).error_node
error_node = self.fix_errornode(error_node)
continue
if isinstance(node, EOS):
lbnode = self.get_languagebox(node)
if self.cursor.node is lbnode:
self.draw_cursor(paint, x, 4 + y * self.fontht)
if lbnode:
color = colors[(lbox-1) % len(colors)]
if lbox > 0:
lbox -= 1
node = lbnode.next_term
highlighter = self.get_highlighter(node)
renderer = self.get_renderer(node)
if self.selected_lbox is lbnode:
# draw bracket
self.draw_lbox_bracket(paint, ']', node, x, y, color)
draw_lbox = False
lbnode = self.get_languagebox(node)
if lbnode and self.selected_lbox is lbnode:
draw_lbox = True
error_node = self.tm.get_parser(lbnode.symbol.ast).error_node
error_node = self.fix_errornode(error_node)
else:
error_node = self.tm.get_mainparser().error_node
error_node = self.fix_errornode(error_node)
continue
else:
self.lines[line].width = x / self.fontwt
break
# draw language boxes
if lbox > 0 and (draw_lbox or draw_all_boxes):
if draw_all_boxes:
color = colors[(lbox-1) % len(colors)]
color.setAlpha(alpha)
else:
color = colors[0]
color.setAlpha(alpha)
if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box
color.setAlpha(20)
renderer.update_image(node)
if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal):
if not node.image or node.plain_mode:
if isinstance(node, BOS) and isinstance(node.next_term, EOS):
self.draw_lbox_hints(paint, node, x, y, color)
else:
paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color)
# prepare selection drawing
if node is selection_start.node:
if node.lookup == "<return>":
sel_x = x
else:
sel_x = x + selection_start.pos * self.fontwt
draw_selection_start = (sel_x, y, line)
if node is selection_end.node:
draw_selection_end = (x + selection_end.pos * self.fontwt, y, line)
# draw node
dx, dy = renderer.paint_node(paint, node, x, y, highlighter)
x += dx
self.lines[line].height = max(self.lines[line].height, dy)
# Draw footnotes and add heatmap data to overlay.
annotes = [annote.annotation for annote in node.get_annotations_with_hint(Heatmap)]
# Heatmap data can always be sent to the overlay, it won't be
# rendered unless the user selects the Heatmap HUD radio button.
for annote in annotes:
self.overlay.add_heatmap_datum(line + 1, annote)
if self.hud_eval or self.hud_types:
infofont = QApplication.instance().tool_info_font
annotes = []
annotes_ = node.get_annotations_with_hint(Footnote)
if self.hud_eval:
for annote in annotes_:
if annote.has_hint(HUDEval):
annotes.append(annote)
elif self.hud_types:
for annote in annotes_:
if annote.has_hint(HUDTypes):
annotes.append(annote)
footnote = " ".join([annote.annotation for annote in annotes])
if footnote.strip() != "":
if not self.tm.tool_data_is_dirty:
infofont.font.setBold(True)
else:
infofont.font.setBold(False)
paint.setFont(infofont.font)
paint.setPen(QPen(QColor((highlighter.get_default_color()))))
start_y = self.fontht + ((y + 1) * self.fontht)
paint.drawText(QtCore.QPointF(x-dx, start_y), footnote)
self.lines[line].height = max(self.lines[line].height, 2)
paint.setFont(self.font)
# after we drew a return, update line information
if node.lookup == "<return>" and not node is first_node:
# draw lbox to end of line
if draw_lbox or (draw_all_boxes and lbox > 0):
paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color)
self.lines[line].width = x / self.fontwt
x = 0
y += self.lines[line].height
line += 1
self.lines[line].height = 1 # reset height
if self.show_highlight_line:
if node.lookup == "<return>" or isinstance(node, BOS):
if self.cursor.line == line:
self.highlight_line(paint, y)
# draw cursor
if node is self.cursor.node and self.show_cursor:
draw_x = max(0, x-dx)
cursor_pos = self.cursor.pos
if node.symbol.name == "\r":
cursor_pos = 0
if node.image and not node.plain_mode:
draw_x = x
cursor_pos = 0
self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 4 + y * self.fontht)
if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor:
draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3)
paint.drawRect(draw_cursor_at)
# set lbox info coordinates
infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht)
draw_cursor = False
# draw squiggly line
if node is error_node or (show_namebinding and self.tm.has_error(node)):
if isinstance(node, EOS):
length = self.fontwt
else:
length = len(node.symbol.name)*self.fontwt
if isinstance(node.symbol, MagicTerminal):
self.draw_vertical_squiggly_line(paint,x,y)
else:
if self.tm.has_error(node):
err_color = "orange"
else:
err_color = "red"
self.draw_squiggly_line(paint,x-length,y,length, err_color)
node = node.next_term
if selection_start != selection_end:
self.draw_selection(paint, draw_selection_start, draw_selection_end, max_y)
# paint infobox
if False:
lang_name = self.tm.parsers[selected_language]
lang_status = self.tm.get_parser[selected_language][0].last_status
if lang_status is True:
color = QColor(100,255,100)
else:
color = QColor(255,100,100)
paint.setFont(infofont)
paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*infofontwt, infofontht), color)
paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name)
paint.setFont(self.font)
return x, y, line
def highlight_line(self, paint, y):
width = self.parentWidget().geometry().width()
paint.fillRect(QRect(-5, y * self.fontht + 3, width, self.fontht), self.highlight_line_color)
def draw_lbox_hints(self, paint, node, x, y, color):
if node is self.cursor.node:
return
alpha = color.alpha()
color.setAlpha(255)
path = QPainterPath()
x = x - 2
y = y*self.fontht + 4
path.moveTo(x, y)
path.lineTo(x+6, y)
path.lineTo(x+3, y+3)
path.lineTo(x, y)
paint.fillPath (path, QBrush (color));
color.setAlpha(alpha)
def draw_lbox_bracket(self, paint, bracket, node, x, y, color):
assert bracket in ['[',']']
oldpen = paint.pen()
newpen = QPen()
color.setAlpha(255)
newpen.setColor(color)
newpen.setWidth(1)
paint.setPen(newpen)
# paint brackets
path = QPainterPath()
if bracket == '[':
if x == 0:
tmpx = x + 2
else:
tmpx = x + 1 # adjust bracket position
path.moveTo(tmpx, 3+y*self.fontht)
path.lineTo(tmpx-2, 3+y*self.fontht)
path.moveTo(tmpx-2, 3+y*self.fontht)
path.lineTo(tmpx-2, 3+y*self.fontht + self.fontht - 1)
else:
tmpx = x - 1
path.moveTo(tmpx, 3+y*self.fontht)
path.lineTo(tmpx+2, 3+y*self.fontht)
path.moveTo(tmpx+2, 3+y*self.fontht)
path.lineTo(tmpx+2, 3+y*self.fontht + self.fontht - 1)
path.lineTo(tmpx, 3+y*self.fontht + self.fontht - 1)
paint.drawPath(path)
paint.setPen(oldpen)
def fix_errornode(self, error_node):
if not error_node:
return
while isinstance(error_node.symbol, IndentationTerminal):
error_node = error_node.next_term
return error_node
def draw_cursor(self, paint, x, y):
pen = paint.pen()
colorhex = self.palette().color(QPalette.Text)
pen.setColor(QColor(colorhex))
paint.setPen(pen)
draw_cursor_at = QRect(x, y, 0, self.fontht - 3)
paint.drawRect(draw_cursor_at)
def draw_vertical_squiggly_line(self, paint, x, y):
paint.setPen(Qt.CustomDashLine)
pen = paint.pen()
pen.setDashPattern([2,2])
pen.setColor(QColor("red"))
paint.setPen(pen)
y = 3+y*self.fontht
paint.drawLine(x-1, y, x-1, y+self.fontht)
paint.drawLine(x, y+2, x, y+self.fontht)
paint.setPen(Qt.SolidLine)
def draw_squiggly_line(self, paint, x, y, length, color):
paint.setPen(Qt.CustomDashLine)
pen = paint.pen()
pen.setDashPattern([2,2])
pen.setColor(QColor(color))
paint.setPen(pen)
y = (y+1)*self.fontht+1
paint.drawLine(x, y, x+length, y)
paint.drawLine(x+2, y+1, x+2+length, y+1)
paint.setPen(Qt.SolidLine)
def draw_selection(self, paint, draw_selection_start, draw_selection_end, max_y):
x1, y1, line1 = draw_selection_start
x2, y2, line2 = draw_selection_end
start = min(self.tm.selection_start, self.tm.selection_end)
end = max(self.tm.selection_start, self.tm.selection_end)
if x1 + y1 + line1 + x2 + y2 + line2 == 0:
# everything out of viewport, draw nothing
# unless start and end are on opposite sides of the viewport
if not(start.line <= self.paint_start[0] and end.line >= self.paint_start[0] + max_y):
return
if x1 + y1 + line1 == 0:
# start outside of viewport
line1 = self.paint_start[0]
if x2 + y2 + line2 == 0:
# end outside of viewport
line2 = self.paint_start[0] + max_y
y2 = max_y
if y1 == y2:
paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100))
else:
width = max(self.fontwt, self.tm.lines[line1].width*self.fontwt)
paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, width - x1, self.fontht), QColor(0,0,255,100))
y = y1 + self.tm.lines[line1].height
for i in range(line1+1, line2):
width = self.tm.lines[i].width*self.fontwt
if width == 0:
width = self.fontwt
paint.fillRect(QRectF(0, 3 + y * self.fontht, width, self.fontht), QColor(0,0,255,100))
y = y + self.tm.lines[i].height
paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100))
def get_highlighter(self, node):
root = node.get_root()
base = lang_dict[self.tm.get_language(root)].base
s = syntaxhighlighter.get_highlighter(base, self.palette())
return s
def get_languagebox(self, node):
root = node.get_root()
lbox = root.get_magicterminal()
return lbox
def get_renderer(self, node):
root = node.get_root()
base = lang_dict[self.tm.get_language(root)].base
return renderers.get_renderer(base, self.fontwt, self.fontht, self.fontd)
def focusNextPrevChild(self, b):
# don't switch to next widget on TAB
return False
def mousePressEvent(self, e):
if e.button() == Qt.LeftButton:
self.tm.input_log.append("# mousePressEvent")
self.coordinate_to_cursor(e.x(), e.y())
self.tm.selection_start = self.tm.cursor.copy()
self.tm.selection_end = self.tm.cursor.copy()
self.tm.input_log.append("self.selection_start = self.cursor.copy()")
self.tm.input_log.append("self.selection_end = self.cursor.copy()")
self.getWindow().showLookahead()
self.update()
def mouseDoubleClickEvent(self, e):
if e.button() == Qt.LeftButton:
self.coordinate_to_cursor(e.x(), e.y())
node = self.tm.get_node_from_cursor()
lbox = self.get_languagebox(node)
if lbox and lbox.symbol.name == "<IPython>":
if lbox.plain_mode is False:
lbox.plain_mode = True
else:
lbox.plain_mode = False
self.update()
return
elif node.image is None:
self.tm.doubleclick_select()
self.update()
return
if node.plain_mode is False:
node.plain_mode = True
self.tm.cursor.pos = len(node.symbol.name)
else:
node.plain_mode = False
self.update()
def cursor_to_coordinate(self):
y = 0
for l in self.tm.lines[:self.cursor.line]:
y += l.height * self.fontht
x = self.tm.cursor.get_x() * self.fontwt
y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht
return (x,y)
def coordinate_to_cursor(self, x, y):
mouse_y = y / self.fontht
first_line = self.paint_start[0]
y_offset = self.paint_start[1]
y = y_offset
line = first_line
if mouse_y < 0:
while line > 0:
y -= self.tm.lines[line].height
if y < mouse_y:
break
line -= 1
else:
while line < len(self.tm.lines) - 1:
y += self.tm.lines[line].height
if y > mouse_y:
break
line += 1
self.tm.cursor.line = line
cursor_x = int(round(float(x) / self.fontwt))
self.tm.cursor.move_to_x(cursor_x)
self.tm.input_log.append("self.cursor.line = %s" % str(line))
self.tm.log_input("cursor.move_to_x", str(cursor_x))
if mouse_y > y or self.tm.cursor.get_x() != cursor_x:
return False
return True
def mouseMoveEvent(self, e):
# apparently this is only called when a mouse button is clicked while
# the mouse is moving
if self.tm.input_log[-2].startswith("self.cursor.move_to_x"):
# only log the last move event
self.tm.input_log.pop()
self.tm.input_log.pop()
self.tm.input_log.pop()
self.coordinate_to_cursor(e.x(), e.y())
self.tm.selection_end = self.tm.cursor.copy()
self.tm.input_log.append("self.selection_end = self.cursor.copy()")
self.update()
self.getEditorTab().keypress()
def keyPressEvent(self, e):
self.timer.start(500)
self.show_cursor = True
startundotimer = False
key = KeyPress(e)
# key presses to ignore
if key.is_modifier or key.page_up or key.page_down:
return
# has been processes in get_nodes_at_pos -> reset
self.edit_rightnode = False
if key.escape:
self.tm.key_escape()
elif key.backspace:
startundotimer = True
self.tm.key_backspace()
elif key.home:
self.tm.key_home(shift=key.m_shift)
elif key.end:
self.tm.key_end(shift=key.m_shift)
elif key.is_arrow:
if key.jump_word:
self.tm.ctrl_cursor(key, shift=key.m_shift)
else:
self.tm.key_cursors(key, shift=key.m_shift)
elif key.delete:
startundotimer = True
self.tm.key_delete()
elif e.key() == Qt.Key_F3:
self.tm.find_next()
# User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't
# sensibly insert into the text.
elif key.has_action_modifier:
pass
# every other normal key press
else:
startundotimer = True
if e.key() == Qt.Key_Tab:
text = " "
else:
# text is a char array, so we need the first letter
# to match it against the set
text = e.text()
if text.isEmpty() or text.toUtf8()[0] not in whitelist:
logging.debug("Key %s not supported" % text)
return
self.tm.key_normal(text)
self.getWindow().btReparse([])
self.update()
self.emit(SIGNAL("keypress(QKeyEvent)"), e)
self.getWindow().showLookahead()
if startundotimer:
self.undotimer.start(500)
def showLanguageBoxMenu(self):
self.showSubgrammarMenu()
self.create_languagebox()
def create_languagebox(self):
if self.sublanguage:
if self.tm.hasSelection():
self.tm.surround_with_languagebox(self.sublanguage)
else:
self.tm.add_languagebox(self.sublanguage)
def change_languagebox(self):
if self.sublanguage:
self.tm.change_languagebox(self.sublanguage)
def showCodeCompletion(self):
l = self.tm.getCompletion()
if l:
self.showCodeCompletionMenu(l)
def println(self, prestring, y):
node = self.lines[y].node.next_term
x = []
while node is not None and node.symbol.name != "\r":
x.append(node.symbol.name)
node = node.next_term
print(prestring, "".join(x))
def print_line(self, y):
current = self.lines[y].node
while True:
print(current)
current = current.next_term
if current is None:
return
# ========================== AST modification stuff ========================== #
def insertTextNoSim(self, text):
self.viewport_y = 0
self.tm.import_file(text)
return
def getTL(self):
return self.getWindow().tl
def getPL(self):
return self.getWindow().pl
def getLRP(self):
return self.getWindow().lrp
def getWindow(self):
return self.window()
def getEditorTab(self):
return self.parent().parent().parent()
def getScrollArea(self):
return self.parent().parent()
def createSubgrammarMenu(self, menu, change=False):
self.sublanguage = None
tmp = None
if change:
# try and find lbox and set cursor to previous node before getting
# lookahead list
root = self.tm.cursor.node.get_root()
lbox = root.get_magicterminal()
if lbox:
tmp = self.tm.cursor.node
self.tm.cursor.node = lbox.prev_term
# Create actions
bf = QFont()
bf.setBold(True)
valid_langs = []
for l in languages:
if "<%s>" % l in self.tm.getLookaheadList():
valid_langs.append(l)
if tmp:
# undo cursor change
self.tm.cursor.node = tmp
if len(valid_langs) > 0:
for l in valid_langs:
item = QAction(str(l), menu)
item.setData(l)
self._set_icon(item, l)
item.setFont(bf)
menu.addAction(item)
menu.addSeparator()
for l in languages:
item = QAction(str(l), menu)
item.setData(l)
self._set_icon(item, l)
menu.addAction(item)
return menu
def showSubgrammarMenu(self):
menu = QtGui.QMenu("Language", self)
self.createSubgrammarMenu(menu)
x,y = self.cursor_to_coordinate()
action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
if action:
self.sublanguage = action.data().toPyObject()
self.edit_rightnode = True
def _set_icon(self, mitem, lang):
if lang.base.lower() == "html":
icon = QIcon.fromTheme("text-xhtml+xml")
else:
icon = QIcon.fromTheme("text-x-" + lang.base.lower())
if icon.isNull():
icon = QIcon.fromTheme("application-x-" + lang.base.lower())
if icon.isNull():
icon = QIcon.fromTheme("text-x-generic")
mitem.setIcon(icon)
def showCodeCompletionMenu(self, l):
menu = QtGui.QMenu( self )
# Create actions
toolbar = QtGui.QToolBar()
for n in l:
path = []
for p in n.path:
if p and p.name:
path.append(p.name)
if n.vartype:
vartype = n.vartype
while vartype.children != []:
try:
vartype = vartype.children[0]
except KeyError:
vartype = vartype.children["name"]
text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind)
elif n.kind == "method":
text = self.cc_method(n) + " - %s" % (".".join(path))
else:
text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind)
item = toolbar.addAction(text, self.createCCFunc(n.name))
item.setIcon(QIcon("gui/" + n.kind + ".png"))
menu.addAction(item)
x,y = self.cursor_to_coordinate()
menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
def cc_method(self, n):
s = [n.name, "("]
param_ln = n.astnode.children["params"]
if isinstance(param_ln, ListNode):
for p in param_ln.children:
tmp = p.children["type"]
if isinstance(tmp, AstNode):
s.append(tmp.children["name"].symbol.name)
else:
s.append(tmp.symbol.name)
s.append(" ")
s.append(p.children["name"].symbol.name)
if p != param_ln.children[-1]:
s.append(", ")
s.append(")")
return "".join(s)
def createMenuFunction(self, l):
def action():
self.sublanguage = l
self.edit_rightnode = True
return action
def createCCFunc(self, text):
def action():
self.tm.pasteCompletion(text)
return action
def selectSubgrammar(self, item):
pass
def saveToJson(self, filename, swap=False):
whitespaces = self.tm.get_mainparser().whitespaces
root = self.tm.parsers[0][0].previous_version.parent
language = self.tm.parsers[0][2]
manager = JsonManager()
manager.save(root, language, whitespaces, filename)
if not swap:
self.tm.changed = False
self.emit(SIGNAL("painted()"))
def loadFromJson(self, filename):
manager = JsonManager()
language_boxes = manager.load(filename)
self.tm = TreeManager()
self.tm.load_file(language_boxes)
self.reset()
def export(self, run=False, profile=False, source=None, debug=False):
return self.tm.export(None, run, profile, source=source, debug=debug)
class BootstrapParser(object):
def __init__(self, lr_type=1, whitespaces=False):
self.lr_type = lr_type
self.whitespaces = whitespaces
# load (old) parser for grammar grammar
self.rules = {}
self.lrules = []
self.start_symbol = None
self.incparser = None
self.inclexer = None
self.terminals = set()
self.extra_alternatives = {}
self.change_startrule = None
self.options = {}
self.precedences = []
def implicit_ws(self):
if self.options.has_key("implicit_ws"):
if self.options["implicit_ws"] == "true":
return True
return False
def indentation_based(self):
if self.options.has_key("indentation"):
if self.options["indentation"] == "true":
return True
return False
def parse(self, ecogrammar):
# this is only called for grammars based on Eco Grammar (not Eco Grammar (Eco))
from grammars.eco_grammar import eco_grammar as grammar
self.lexer = IncrementalLexer(grammar.priorities)
self.parser = IncParser(grammar.grammar, 1, True)
self.parser.init_ast()
self.ast = self.parser.previous_version.parent
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, grammar.name)
self.treemanager.import_file(ecogrammar)
if self.parser.last_status == False:
raise Exception("Invalid input grammar: at %s %s" % (self.parser.error_node.prev_term, self.parser.error_node))
self.read_options()
self.create_parser()
self.create_lexer()
def read_options(self):
startrule = self.ast.children[1] # startrule
assert startrule.symbol.name == "Startrule"
grammar = startrule.children[1]
assert grammar.symbol.name == "grammar"
for element in grammar.children:
if element.symbol.name == "options":
break
if element.symbol.name != "options":
# grammar has no options
print("warning: grammar has no options")
# backwards compatibility
if self.whitespaces:
self.options["implicit_ws"] = "true"
return
options = element
assert options.symbol.name == "options"
self.parse_options(options)
def parse_options(self, options):
if options.children == []:
return
if len(options.children) > 0:
assert options.children[0].symbol.name == "settings"
self.parse_settings(options.children[0])
if len(options.children) > 1:
assert options.children[1].symbol.name == "precedences"
self.parse_precedences(options.children[1])
def parse_settings(self, options):
if options.children == []:
return
if len(options.children) == 2:
more = options.children[0]
self.parse_settings(more)
option = options.children[1]
else:
option = options.children[0]
name = option.children[2].symbol.name
choice = option.children[6]
assert choice.symbol.name == "choice"
self.options[name] = choice.children[0].symbol.name
def parse_precedences(self, precedences):
if precedences.children == []:
return
# recursively parse other precedences
if len(precedences.children) == 2:
more = precedences.children[0]
self.parse_precedences(more)
precedence = precedences.children[1]
else:
precedence = precedences.children[0]
# parse single precedence
name = precedence.children[0].symbol.name
terminals = self.parse_precedence_symbols(precedence.children[2])
self.precedences.append((name, terminals))
def parse_precedence_symbols(self, symbol):
s = []
for c in symbol.children:
if c.symbol.name == "WS":
continue
if c.symbol.name == "terminals":
rec_s = self.parse_precedence_symbols(symbol.children[0])
s.extend(rec_s)
if c.lookup == "terminal":
s.append(c.symbol.name[1:-1])
return s
def create_parser(self, pickle_id = None):
startrule = self.ast.children[1] # startrule
grammar = startrule.children[1]
parser = grammar.children[0]
assert parser.symbol.name == "parser"
self.parse_rules(parser)
if self.implicit_ws():
ws_rule = Rule()
ws_rule.symbol = Nonterminal("WS")
ws_rule.add_alternative([Terminal("<ws>"), Nonterminal("WS")])
ws_rule.add_alternative([Terminal("<return>"), Nonterminal("WS")])
ws_rule.add_alternative([Terminal("<backslash>"), Terminal("<return>"), Nonterminal("WS")])
ws_rule.add_alternative([]) # or empty
self.rules[ws_rule.symbol] = ws_rule
# allow whitespace/comments at beginning of file
start_rule = Rule()
start_rule.symbol = Nonterminal("Startrule")
start_rule.add_alternative([Nonterminal("WS"), self.start_symbol])
self.rules[start_rule.symbol] = start_rule
self.start_symbol = start_rule.symbol
incparser = IncParser()
incparser.from_dict(self.rules, self.start_symbol, self.lr_type, self.implicit_ws(), pickle_id, self.precedences)
incparser.init_ast()
self.incparser = incparser
def parse_rules(self, node):
if node.children[0].symbol.name == "parser":
self.parse_rules(node.children[0])
self.parse_rule(node.children[3])
elif node.children[0].symbol.name == "rule":
self.parse_rule(node.children[0])
def parse_rule(self, node):
name = node.children[0].symbol.name
alternatives = self.parse_alternatives(node.children[4])
symbol = Nonterminal(name)
if self.start_symbol is None:
self.start_symbol = symbol
if self.change_startrule and symbol.name == self.change_startrule:
self.start_symbol = symbol
r = Rule(symbol)
for a in alternatives:
r.add_alternative(a[0], a[1], a[2])
# add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes)
if self.extra_alternatives.has_key(symbol.name):
for n in self.extra_alternatives[symbol.name]:
r.add_alternative([MagicTerminal(n), Nonterminal("WS")], None)
self.rules[symbol] = r
def parse_alternatives(self, node):
if node.children[0].symbol.name == "alternatives":
alternatives = self.parse_alternatives(node.children[0])
alternative = self.parse_alternative(node.children[3])
alternatives.append(alternative)
return alternatives
elif node.children[0].symbol.name == "right":
return [self.parse_alternative(node.children[0])]
def parse_alternative(self, node):
if len(node.children) > 0:
annotation = None
prec = None
for c in node.children:
if c.symbol.name == "symbols":
symbols = self.parse_symbols(c)
if c.symbol.name == "prec":
prec = self.parse_prec(c)
if c.symbol.name == "annotations":
annotation = self.parse_annotation(c)
return (symbols, annotation, prec)
else:
return ([], None, None)
def parse_prec(self, node):
if node.children:
c = node.children[2]
return c.symbol.name[1:-1]
def parse_symbols(self, node):
if node.children[0].symbol.name == "symbols":
symbols = self.parse_symbols(node.children[0])
symbol = self.parse_symbol(node.children[1])
symbols.append(symbol)
if (isinstance(symbol, Terminal) or isinstance(symbol, MagicTerminal)) and self.implicit_ws():
symbols.append(Nonterminal("WS"))
return symbols
elif node.children[0].symbol.name == "symbol":
l = []
symbol = self.parse_symbol(node.children[0])
l.append(symbol)
if isinstance(symbol, Terminal) and self.implicit_ws():
l.append(Nonterminal("WS"))
return l
def parse_symbol(self, node):
node = node.children[0]
if node.lookup == "nonterminal":
return Nonterminal(node.symbol.name)
elif node.lookup == "terminal":
self.terminals.add(node.symbol.name[1:-1])
return Terminal(node.symbol.name[1:-1])
elif node.lookup == "languagebox":
return MagicTerminal(node.symbol.name)
def parse_annotation(self, node):
a_options = node.children[2]
assert a_options.symbol.name == "a_options"
if a_options.children[0].symbol.name == "astnode":
return self.parse_astnode(a_options.children[0])
elif a_options.children[0].symbol.name == "expression":
return self.parse_expression(a_options.children[0])
elif a_options.children[0].symbol.name == "forloop":
return self.parse_foreach(a_options.children[0])
def parse_astnode(self, node):
name = node.children[0].symbol.name
children = self.parse_astnode_children(node.children[4])
d = {}
for n, expr in children:
d[n] = expr
return AstNode(name, d)
def parse_astnode_children(self, node):
assert node.symbol.name == "astnode_children"
if node.children[0].symbol.name == "astnode_child":
return [self.parse_astnode_child(node.children[0])]
elif node.children[0].symbol.name == "astnode_children":
children = self.parse_astnode_children(node.children[0])
child = self.parse_astnode_child(node.children[3])
children.append(child)
return children
def parse_astnode_child(self, node):
assert node.symbol.name == "astnode_child"
name = node.children[0].symbol.name
if node.children[4].symbol.name == "expression":
expr = self.parse_expression(node.children[4])
elif node.children[4].symbol.name == "reference":
expr = self.parse_reference(node.children[4])
return (name, expr)
def parse_expression(self, node):
if node.children[0].symbol.name == "node":
return self.parse_node(node.children[0])
elif node.children[0].symbol.name == "list":
return self.parse_list(node.children[0])
elif node.children[0].symbol.name == "node_ref":
return self.parse_noderef(node.children[0])
else:
expr1 = self.parse_expression(node.children[0])
if node.children[3].symbol.name == "node":
expr2 = self.parse_node(node.children[3])
else:
expr2 = self.parse_list(node.children[3])
return AddExpr(expr1, expr2)
def parse_foreach(self, node):
item = self.parse_node(node.children[4])
expr = self.parse_astnode(node.children[7])
return Foreach(node.symbol.name, item, expr)
def parse_noderef(self, node):
lookup = self.parse_node(node.children[0])
attr = node.children[3]
lookup.attribute = attr.symbol.name
return lookup
def parse_node(self, node):
return LookupExpr(int(node.children[2].symbol.name))
def parse_list(self, node):
return ListExpr(self.parse_listloop(node.children[2]))
def parse_reference(self, node):
base = node.children[0].symbol.name
ref = node.children[4].symbol.name
return ReferenceExpr(base, ref)
def parse_listloop(self, node):
if len(node.children) == 0:
return []
if node.children[0].symbol.name == "list_loop":
l = self.parse_listloop(node.children[0])
element = self.parse_unknown(node.children[3])
l.append(element)
return l
else:
return [self.parse_unknown(node.children[0])]
def parse_unknown(self, node):
if node.symbol.name == "node":
return self.parse_node(node)
elif node.symbol.name == "astnode":
return self.parse_astnode(node)
def create_lexer(self):
startrule = self.ast.children[1] # startrule
grammar = startrule.children[1]
for element in grammar.children:
if element.symbol.name == "lexer":
break
lexer = element
assert lexer.symbol.name == "lexer"
self.parse_lexer(lexer)
names = []
regexs = []
for name, regex in self.lrules:
names.append(name)
regexs.append(regex)
# add so far undefined terminals
undefined_terminals = self.terminals.difference(set(names))
import re
for t in undefined_terminals:
names.insert(0, t)
regexs.insert(0,re.escape(t))
self.inclexer = IncrementalLexerCF()
self.inclexer.from_name_and_regex(names, regexs)
if self.indentation_based():
self.inclexer.indentation_based = True
def parse_lexer(self, lexer):
if lexer.children[0].symbol.name == "lrule":
self.parse_lrule(lexer.children[0])
elif lexer.children[0].symbol.name == "lexer":
self.parse_lexer(lexer.children[0])
self.parse_lrule(lexer.children[1])
def parse_lrule(self, lrule):
assert lrule.children[0].symbol.name == "tokenname"
name = lrule.children[0].children[0].symbol.name
regex = lrule.children[3].symbol.name[1:-1]
self.lrules.append((name, regex))
class NodeEditor(QFrame):
# ========================== init stuff ========================== #
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent)
self.infofont = QtGui.QFont('Courier', 6)
self.infofontht = QtGui.QFontMetrics(self.infofont).height() + 3
self.infofontwt = QtGui.QFontMetrics(self.infofont).width(" ")
self.viewport_y = 0 # top visible line
self.imagemode = False
self.image = None
self.scroll_height = 0
self.scroll_width = 0
self.timer = QTimer(self)
self.backuptimer = QTimer(self)
self.connect(self.timer, SIGNAL("timeout()"), self.analysis_timer)
self.connect(self.backuptimer, SIGNAL("timeout()"), self.backup_timer)
self.backuptimer.start(30000)
self.undotimer = QTimer(self)
self.connect(self.undotimer, SIGNAL("timeout()"), self.trigger_undotimer)
self.blinktimer = QTimer(self)
self.blinktimer.start(500)
self.connect(self.blinktimer, SIGNAL("timeout()"), self.trigger_blinktimer)
self.show_cursor = True
self.lightcolors = [QColor("#333333"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")]
self.darkcolors = [QColor("#999999"), QColor("#859900"), QColor("#DC322F"), QColor("#268BD2"), QColor("#D33682"), QColor("#B58900"), QColor("#2AA198")]
self.setCursor(Qt.IBeamCursor)
def analysis_timer(self):
if self.getWindow().show_namebinding():
self.tm.analyse()
self.update()
self.timer.stop()
# save swap
filename = self.getEditorTab().filename
if filename:
self.saveToJson(filename + ".swp", True)
def backup_timer(self):
filename = self.getEditorTab().filename
if filename:
self.saveToJson(filename + ".bak", True)
def trigger_blinktimer(self):
if self.timer.isActive():
self.show_cursor = True
return
self.show_cursor ^= True
self.update()
def trigger_undotimer(self):
self.tm.save_current_version()
self.undotimer.stop()
def setImageMode(self, boolean):
self.imagemode = boolean
def reset(self):
#self.getWindow().ui.scrollArea.horizontalScrollBar().setValue(0)
#self.getWindow().ui.scrollArea.verticalScrollBar().setValue(0)
self.update()
def set_mainlanguage(self, parser, lexer, lang_name):
self.tm = TreeManager()
self.tm.add_parser(parser, lexer, lang_name)
def set_sublanguage(self, language):
self.sublanguage = language
def event(self, event):
if event.type() == QEvent.ToolTip:
if QToolTip.isVisible():
QToolTip.hideText()
event.ignore()
return True
pos = event.pos()
temp_cursor = self.tm.cursor.copy()
result = self.coordinate_to_cursor(pos.x(), pos.y())
node = self.tm.cursor.node
self.tm.cursor.line = temp_cursor.line
self.tm.cursor.node = temp_cursor.node
self.tm.cursor.pos = temp_cursor.pos
if not result:
event.ignore()
return True
msg = self.tm.get_error(node)
if not msg:
msg = self.tm.get_error(node)
if msg:
QToolTip.showText(event.globalPos(), msg);
return True
return QFrame.event(self, event)
# ========================== GUI related stuff ========================== #
def blink(self):
if self.show_cursor:
self.show_cursor = 0
else:
self.show_cursor = 1
self.update()
def sliderChanged(self, value):
change = self.viewport_y - value
self.viewport_y = value
self.update()
def sliderXChanged(self, value):
self.move(-value*self.fontwt,0)
self.resize(self.parentWidget().geometry().width() + value*self.fontwt, self.geometry().height())
if self.x() == 0:
self.updateGeometry()
self.update()
def getScrollSizes(self):
total_lines = 0
max_width = 0
for l in self.lines:
total_lines += l.height
max_width = max(max_width, l.width)
max_visible_lines = self.geometry().height() / self.fontht
self.scroll_height = max(0, total_lines - max_visible_lines)
current_width = self.parentWidget().geometry().width() / self.fontwt
self.scroll_width = max(0, max_width - current_width)
def paintEvent(self, event):
gfont = QApplication.instance().gfont
self.font = gfont.font
self.fontwt = gfont.fontwt
self.fontht = gfont.fontht
QtGui.QFrame.paintEvent(self, event)
paint = QtGui.QPainter()
if self.imagemode:
self.image = QImage()
paint.begin(self.image)
else:
paint.begin(self)
paint.setFont(self.font)
y = 0
x = 0
self.longest_column = 0
# calculate how many lines we need to show
self.init_height = self.geometry().height()
self.paintLines(paint, self.viewport_y)
paint.end()
total_lines = 0
max_width = 0
for l in self.lines:
total_lines += l.height
max_width = max(max_width, l.width)
max_visible_lines = self.geometry().height() / self.fontht
self.scroll_height = max(0, total_lines - max_visible_lines)
current_width = self.parentWidget().geometry().width() / self.fontwt
self.scroll_width = max(0, max_width - current_width)
self.emit(SIGNAL("painted()"))
# paint lines using new line manager
def paintLines(self, paint, startline):
# find internal line corresponding to visual line
visual_line = 0
internal_line = 0
for l in self.tm.lines:
if visual_line + l.height > startline:
break
visual_line += l.height
internal_line += 1
x = 0
y = visual_line - startline # start drawing outside of viewport to display partial images
self.paint_start = (internal_line, y)
max_y = self.geometry().height() / self.fontht
line = internal_line
node = self.tm.lines[line].node
self.paint_nodes(paint, node, x, y, line, max_y)
#XXX if starting node is inside language box, init lbox with amout of languge boxes
def paint_nodes(self, paint, node, x, y, line, max_y, lbox=0):
settings = QSettings("softdev", "Eco")
if settings.value("app_theme", "Light").toString() == "Dark":
alpha = 100
colors = self.darkcolors
else:
alpha = 40
colors = self.lightcolors
first_node = node
selected_language = self.tm.mainroot
error_node = self.tm.get_mainparser().error_node
error_node = self.fix_errornode(error_node)
highlighter = self.get_highlighter(node)
selection_start = min(self.tm.selection_start, self.tm.selection_end)
selection_end = max(self.tm.selection_start, self.tm.selection_end)
draw_selection_start = (0,0,0)
draw_selection_end = (0,0,0)
start_lbox = self.get_languagebox(node)
editor = self.get_editor(node)
self.selected_lbox = self.tm.get_languagebox(self.tm.cursor.node)
#XXX get initial x for langbox
if start_lbox:
lbox += 1
if start_lbox and self.selected_lbox is start_lbox:
draw_lbox = True
else:
draw_lbox = False
draw_all_boxes = self.getWindow().show_languageboxes()
self.lines = self.tm.lines
self.cursor = self.tm.cursor
self.lines[line].height = 1 # reset height
draw_cursor = True
#l_x = [0]
while y < max_y:
# if we found a language box, continue drawing inside of it
if isinstance(node.symbol, MagicTerminal):
#l_x.append(x)
#node.pos = x
lbox += 1
lbnode = node.symbol.ast
if self.selected_lbox is node:
draw_lbox = True
selected_language = lbnode
else:
draw_lbox = False
node = lbnode.children[0]
highlighter = self.get_highlighter(node)
editor = self.get_editor(node)
error_node = self.tm.get_parser(lbnode).error_node
error_node = self.fix_errornode(error_node)
continue
if isinstance(node, EOS):
lbnode = self.get_languagebox(node)
if self.cursor.node is lbnode:
self.draw_cursor(paint, x, 5 + y * self.fontht)
if lbnode:
#l_x.pop()
if lbox > 0:
lbox -= 1
node = lbnode.next_term
highlighter = self.get_highlighter(node)
editor = self.get_editor(node)
if self.selected_lbox is lbnode:
draw_lbox = False
lbnode = self.get_languagebox(node)
if lbnode and self.selected_lbox is lbnode:
draw_lbox = True
error_node = self.tm.get_parser(lbnode.symbol.ast).error_node
error_node = self.fix_errornode(error_node)
else:
error_node = self.tm.get_mainparser().error_node
error_node = self.fix_errornode(error_node)
continue
else:
self.lines[line].width = x / self.fontwt
break
# draw language boxes
if lbox > 0 and (draw_lbox or draw_all_boxes):
#color = self.nesting_colors[lbox % 5]
if draw_all_boxes:
color = colors[(lbox-1) % len(colors)]
color.setAlpha(alpha)
else:
color = colors[0]
color.setAlpha(alpha)
if draw_lbox and draw_all_boxes: # we are drawing the currently selected language box
color = colors[-1] # overwrite color
color.setAlpha(alpha)
editor.update_image(node)
if node.symbol.name != "\r" and not isinstance(node.symbol, IndentationTerminal):
if not node.image or node.plain_mode:
paint.fillRect(QRectF(x,3 + y*self.fontht, len(node.symbol.name)*self.fontwt, self.fontht), color)
# prepare selection drawing
if node is selection_start.node:
if node.lookup == "<return>":
sel_x = x
else:
sel_x = x + selection_start.pos * self.fontwt
draw_selection_start = (sel_x, y, line)
if node is selection_end.node:
draw_selection_end = (x + selection_end.pos * self.fontwt, y, line)
# draw node
dx, dy = editor.paint_node(paint, node, x, y, highlighter)
x += dx
#y += dy
self.lines[line].height = max(self.lines[line].height, dy)
# after we drew a return, update line information
if node.lookup == "<return>" and not node is first_node:
# draw lbox to end of line
if draw_lbox or (draw_all_boxes and lbox > 0):
paint.fillRect(QRectF(x,3+y*self.fontht, self.geometry().width()-x, self.fontht), color)
self.lines[line].width = x / self.fontwt
x = 0#l_x[-1]
y += self.lines[line].height
line += 1
self.lines[line].height = 1 # reset height
# draw cursor
if node is self.cursor.node and self.show_cursor:
draw_x = max(0, x-dx)
cursor_pos = self.cursor.pos
if node.symbol.name == "\r":
cursor_pos = 0
if node.image and not node.plain_mode:
draw_x = x
cursor_pos = 0
self.draw_cursor(paint, draw_x + cursor_pos * self.fontwt, 5 + y * self.fontht)
if False and line == self.cursor.y and x/self.fontwt >= self.cursor.x and draw_cursor:
draw_cursor_at = QRect(0 + self.cursor.x * self.fontwt, 5 + y * self.fontht, 0, self.fontht - 3)
paint.drawRect(draw_cursor_at)
# set lbox info coordinates
infobox_coordinates = (self.cursor.x * self.fontwt, (y+1) * self.fontht)
draw_cursor = False
# draw squiggly line
if node is error_node or (self.getWindow().show_namebinding() and self.tm.has_error(node)):
if isinstance(node, EOS):
length = self.fontwt
else:
length = len(node.symbol.name)*self.fontwt
if isinstance(node.symbol, MagicTerminal):
self.draw_vertical_squiggly_line(paint,x,y)
else:
if self.tm.has_error(node):
color = "orange"
else:
color = "red"
self.draw_squiggly_line(paint,x-length,y,length, color)
node = node.next_term
if selection_start != selection_end:
self.draw_selection(paint, draw_selection_start, draw_selection_end)
# paint infobox
if False:
lang_name = self.parser_langs[selected_language]
lang_status = self.parsers[selected_language].last_status
if lang_status is True:
color = QColor(100,255,100)
else:
color = QColor(255,100,100)
paint.setFont(self.infofont)
paint.fillRect(QRect(infobox_coordinates[0], 5 + infobox_coordinates[1], len(lang_name)*self.infofontwt, self.infofontht), color)
paint.drawText(QtCore.QPointF(infobox_coordinates[0], -3 + self.fontht + infobox_coordinates[1]), lang_name)
paint.setFont(self.font)
return x, y, line
def fix_errornode(self, error_node):
if not error_node:
return
while isinstance(error_node.symbol, IndentationTerminal):
error_node = error_node.prev_term
return error_node
def draw_cursor(self, paint, x, y):
pen = paint.pen()
colorhex = self.palette().color(QPalette.Text)
pen.setColor(QColor(colorhex))
paint.setPen(pen)
draw_cursor_at = QRect(x, y, 0, self.fontht - 3)
paint.drawRect(draw_cursor_at)
def draw_vertical_squiggly_line(self, paint, x, y):
paint.setPen(Qt.CustomDashLine)
pen = paint.pen()
pen.setDashPattern([2,2])
pen.setColor(QColor("red"))
paint.setPen(pen)
y = 3+y*self.fontht
paint.drawLine(x-1, y, x-1, y+self.fontht)
paint.drawLine(x, y+2, x, y+self.fontht)
paint.setPen(Qt.SolidLine)
def draw_squiggly_line(self, paint, x, y, length, color):
paint.setPen(Qt.CustomDashLine)
pen = paint.pen()
pen.setDashPattern([2,2])
pen.setColor(QColor(color))
paint.setPen(pen)
#x -= length
y = (y+1)*self.fontht+1
paint.drawLine(x, y, x+length, y)
paint.drawLine(x+2, y+1, x+2+length, y+1)
paint.setPen(Qt.SolidLine)
def draw_selection(self, paint, draw_selection_start, draw_selection_end):
x1, y1, line1 = draw_selection_start
x2, y2, line2 = draw_selection_end
if y1 == y2:
paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, x2-x1, self.fontht), QColor(0,0,255,100))
else:
paint.fillRect(QRectF(x1, 3 + y1 * self.fontht, self.tm.lines[line1].width*self.fontwt - x1, self.fontht), QColor(0,0,255,100))
y = y1 + self.tm.lines[line1].height
for i in range(line1+1, line2):
paint.fillRect(QRectF(0, 3 + y * self.fontht, self.tm.lines[i].width*self.fontwt, self.fontht), QColor(0,0,255,100))
y = y + self.tm.lines[i].height
paint.fillRect(QRectF(0, 3 + y2 * self.fontht, x2, self.fontht), QColor(0,0,255,100))
def get_highlighter(self, node):
root = node.get_root()
base = lang_dict[self.tm.get_language(root)].base
s = syntaxhighlighter.get_highlighter(base, self.palette())
return s
def get_languagebox(self, node):
root = node.get_root()
lbox = root.get_magicterminal()
return lbox
def get_editor(self, node):
root = node.get_root()
base = lang_dict[self.tm.get_language(root)].base
return editor.get_editor(base, self.fontwt, self.fontht)
def focusNextPrevChild(self, b):
# don't switch to next widget on TAB
return False
def mousePressEvent(self, e):
if e.button() == Qt.LeftButton:
self.tm.input_log.append("# mousePressEvent")
self.coordinate_to_cursor(e.x(), e.y())
# self.tm.cursor = cursor
self.tm.selection_start = self.tm.cursor.copy()
self.tm.selection_end = self.tm.cursor.copy()
self.tm.input_log.append("self.selection_start = self.cursor.copy()")
self.tm.input_log.append("self.selection_end = self.cursor.copy()")
#self.tm.fix_cursor_on_image()
self.getWindow().showLookahead()
self.update()
def mouseDoubleClickEvent(self, e):
if e.button() == Qt.LeftButton:
self.coordinate_to_cursor(e.x(), e.y())
node = self.tm.get_node_from_cursor()
lbox = self.get_languagebox(node)
if lbox and lbox.symbol.name == "<IPython>":
if lbox.plain_mode is False:
lbox.plain_mode = True
else:
lbox.plain_mode = False
self.update()
return
elif node.image is None:
self.tm.selection_start = self.tm.cursor.copy()
self.tm.selection_start.node = self.tm.cursor.find_previous_visible(self.tm.cursor.node)
self.tm.selection_start.pos = len(self.tm.selection_start.node.symbol.name)
self.tm.selection_end = self.tm.cursor.copy()
self.tm.selection_end.pos = len(self.tm.selection_end.node.symbol.name)
self.tm.cursor.pos = self.tm.selection_end.pos
self.tm.cursor.node = self.tm.selection_end.node
self.update()
return
if node.plain_mode is False:
node.plain_mode = True
self.tm.cursor.pos = len(node.symbol.name)
else:
node.plain_mode = False
self.update()
def cursor_to_coordinate(self):
y = 0
for l in self.tm.lines[:self.cursor.line]:
y += l.height * self.fontht
x = self.tm.cursor.get_x() * self.fontwt
y = y - self.getScrollArea().verticalScrollBar().value() * self.fontht
return (x,y)
def coordinate_to_cursor(self, x, y):
mouse_y = y / self.fontht
first_line = self.paint_start[0]
y_offset = self.paint_start[1]
y = y_offset
line = first_line
while line < len(self.tm.lines) - 1:
y += self.tm.lines[line].height
if y > mouse_y:
break
line += 1
self.tm.cursor.line = line
cursor_x = int(round(float(x) / self.fontwt))
self.tm.cursor.move_to_x(cursor_x, self.tm.lines)
self.tm.input_log.append("self.cursor.line = %s" % str(line))
self.tm.log_input("cursor.move_to_x", str(cursor_x), "self.lines")
if mouse_y > y or self.tm.cursor.get_x() != cursor_x:
return False
return True
def mouseMoveEvent(self, e):
# apparaently this is only called when a mouse button is clicked while
# the mouse is moving
if self.tm.input_log[-2].startswith("self.cursor.move_to_x"):
# only log the last move event
self.tm.input_log.pop()
self.tm.input_log.pop()
self.tm.input_log.pop()
self.coordinate_to_cursor(e.x(), e.y())
self.tm.selection_end = self.tm.cursor.copy()
self.tm.input_log.append("self.selection_end = self.cursor.copy()")
self.update()
def key_to_string(self, key):
if key == Qt.Key_Up:
return "up"
if key == Qt.Key_Down:
return "down"
if key == Qt.Key_Left:
return "left"
if key == Qt.Key_Right:
return "right"
def keyPressEvent(self, e):
startundotimer = False
self.timer.start(500)
self.show_cursor = True
if e.key() in [Qt.Key_Shift, Qt.Key_Alt, Qt.Key_Control, Qt.Key_Meta, Qt.Key_AltGr]:
if e.key() == Qt.Key_Shift:
self.tm.key_shift()
return
text = e.text()
self.edit_rightnode = False # has been processes in get_nodes_at_pos -> reset
if e.key() == Qt.Key_Escape:
self.tm.key_escape()
elif e.key() == Qt.Key_Backspace:
startundotimer = True
self.tm.key_backspace()
elif e.key() in [Qt.Key_Up, Qt.Key_Down, Qt.Key_Left, Qt.Key_Right]:
if e.modifiers() == Qt.ShiftModifier:
self.tm.key_cursors(self.key_to_string(e.key()), True)
elif e.modifiers() == Qt.ControlModifier:
self.tm.ctrl_cursor(self.key_to_string(e.key()))
else:
self.tm.key_cursors(self.key_to_string(e.key()), False)
elif e.key() == Qt.Key_Home:
self.tm.key_home(e.modifiers() == Qt.ShiftModifier)
elif e.key() == Qt.Key_End:
self.tm.key_end(e.modifiers() == Qt.ShiftModifier)
elif e.key() == Qt.Key_Delete:
startundotimer = True
self.tm.key_delete()
elif e.key() == Qt.Key_F3:
self.tm.find_next()
elif e.key() in [Qt.Key_PageUp, Qt.Key_PageDown]:
pass # ignore those keys
elif e.modifiers() in [Qt.ControlModifier, Qt.AltModifier]:
# User pressed Ctrl- Or Alt- (etc.) i.e. a character we can't
# sensibly insert into the text.
pass
else:
startundotimer = True
if e.key() == Qt.Key_Tab:
text = " "
else:
text = e.text()
if text.toUtf8() not in whitelist:
logging.debug("Key %s not supported" % text)
return
self.tm.key_normal(text)
self.getWindow().btReparse([])
self.update()
self.emit(SIGNAL("keypress(QKeyEvent)"), e)
self.getWindow().showLookahead()
if startundotimer:
self.undotimer.start(500)
def showLanguageBoxMenu(self):
self.showSubgrammarMenu()
self.create_languagebox()
def create_languagebox(self):
if self.sublanguage:
if self.tm.hasSelection():
self.tm.surround_with_languagebox(self.sublanguage)
else:
self.tm.add_languagebox(self.sublanguage)
def change_languagebox(self):
if self.sublanguage:
self.tm.change_languagebox(self.sublanguage)
def showCodeCompletion(self):
l = self.tm.getCompletion()
if l:
self.showCodeCompletionMenu(l)
def println(self, prestring, y):
node = self.lines[y].node.next_term
x = []
while node is not None and node.symbol.name != "\r":
x.append(node.symbol.name)
node = node.next_term
print(prestring, "".join(x))
def print_line(self, y):
current = self.lines[y].node
while True:
print(current)
current = current.next_term
if current is None:
return
# ========================== AST modification stuff ========================== #
def insertTextNoSim(self, text):
self.viewport_y = 0
self.tm.import_file(text)
return
def getTL(self):
return self.getWindow().tl
def getPL(self):
return self.getWindow().pl
def getLRP(self):
return self.getWindow().lrp
def getWindow(self):
return self.window()
def getEditorTab(self):
return self.parent().parent().parent()
def getScrollArea(self):
return self.parent().parent()
def createSubgrammarMenu(self, menu, change=False):
self.sublanguage = None
tmp = None
if change:
# try and find lbox and set cursor to previous node before getting
# lookahead list
root = self.tm.cursor.node.get_root()
lbox = root.get_magicterminal()
if lbox:
tmp = self.tm.cursor.node
self.tm.cursor.node = lbox.prev_term
# Create actions
bf = QFont()
bf.setBold(True)
valid_langs = []
for l in languages:
if "<%s>" % l in self.tm.getLookaheadList():
valid_langs.append(l)
if tmp:
# undo cursor change
self.tm.cursor.node = tmp
if len(valid_langs) > 0:
for l in valid_langs:
item = QAction(str(l), menu)
item.setData(l)
self._set_icon(item, l)
item.setFont(bf)
menu.addAction(item)
menu.addSeparator()
for l in languages:
item = QAction(str(l), menu)
item.setData(l)
self._set_icon(item, l)
menu.addAction(item)
return menu
def showSubgrammarMenu(self):
menu = QtGui.QMenu("Language", self)
self.createSubgrammarMenu(menu)
x,y = self.cursor_to_coordinate()
action = menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
if action:
self.sublanguage = action.data().toPyObject()
self.edit_rightnode = True
def _set_icon(self, mitem, lang):
if lang.base.lower() == "html":
icon = QIcon.fromTheme("text-xhtml+xml")
else:
icon = QIcon.fromTheme("text-x-" + lang.base.lower())
if icon.isNull():
icon = QIcon.fromTheme("application-x-" + lang.base.lower())
if icon.isNull():
icon = QIcon.fromTheme("text-x-generic")
mitem.setIcon(icon)
def showCodeCompletionMenu(self, l):
menu = QtGui.QMenu( self )
# Create actions
toolbar = QtGui.QToolBar()
for n in l:
path = []
for p in n.path:
if p and p.name:
path.append(p.name)
if n.vartype:
vartype = n.vartype
while vartype.children != []:
try:
vartype = vartype.children[0]
except KeyError:
vartype = vartype.children["name"]
text = "%s : %s - %s (%s)" % (n.name, vartype.symbol.name, ".".join(path), n.kind)
elif n.kind == "method":
text = self.cc_method(n) + " - %s" % (".".join(path))
else:
text = "%s - %s (%s)" % (n.name, ".".join(path), n.kind)
item = toolbar.addAction(text, self.createCCFunc(n.name))
item.setIcon(QIcon("gui/" + n.kind + ".png"))
menu.addAction(item)
x,y = self.cursor_to_coordinate()
menu.exec_(self.mapToGlobal(QPoint(0,0)) + QPoint(3 + x, y + self.fontht))
def cc_method(self, n):
s = [n.name, "("]
param_ln = n.astnode.children["params"]
if isinstance(param_ln, ListNode):
for p in param_ln.children:
tmp = p.children["type"]
if isinstance(tmp, AstNode):
s.append(tmp.children["name"].symbol.name)
else:
s.append(tmp.symbol.name)
s.append(" ")
s.append(p.children["name"].symbol.name)
if p != param_ln.children[-1]:
s.append(", ")
s.append(")")
return "".join(s)
def createMenuFunction(self, l):
def action():
self.sublanguage = l
self.edit_rightnode = True
return action
def createCCFunc(self, text):
def action():
self.tm.pasteCompletion(text)
return action
def selectSubgrammar(self, item):
pass
def saveToJson(self, filename, swap=False):
whitespaces = self.tm.get_mainparser().whitespaces
root = self.tm.parsers[0][0].previous_version.parent
language = self.tm.parsers[0][2]
manager = JsonManager()
manager.save(root, language, whitespaces, filename)
if not swap:
self.tm.changed = False
self.emit(SIGNAL("painted()"))
def loadFromJson(self, filename):
manager = JsonManager()
language_boxes = manager.load(filename)
self.tm = TreeManager()
self.tm.load_file(language_boxes)
self.reset()
def export(self, run=False):
return self.tm.export(None, run)
def random_deletion(self):
"""Delete random characters within a program."""
print "Running random_deletion on {}".format(self.filename)
program = self.program
self.treemanager.import_file(program)
assert self.parser.last_status == True
self.text_compare(program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
random_lines = random_lines[:
20] # restrict to 20 lines to reduce runtime
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.treemanager.cursor_reset()
self.log.append("self.treemanager.cursor_reset()")
self.move(DOWN, linenr)
self.log.append("self.move(DOWN, %s)" % linenr)
self.move(RIGHT, col)
self.log.append("self.move(RIGHT, %s)" % col)
self.log.append("self.treemanager.key_delete()")
x = self.treemanager.key_delete()
if x == "eos":
continue
self.treemanager.undo_snapshot()
self.log.append("self.treemanager.undo_snapshot()")
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(program)
t1 = TreeManager()
parser, lexer = self.lang.load()
parser.init_ast()
t1.add_parser(parser, lexer, self.lang.name)
t1.set_font_test(7, 17)
t1.import_file(self.program)
assert self.parser.last_status == True
assert parser.last_status == True
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
def random_insertdelete(self):
"""Insert and delete random characters at random locations within a
program."""
print "Running random_insertdelete on {}".format(self.filename)
self.reset()
self.treemanager.import_file(self.program)
assert self.parser.last_status == True
self.text_compare(self.program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
random_lines = random_lines[:
20] # restrict to 20 lines to reduce runtime
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.log.append("self.treemanager.cursor_reset()")
self.log.append("self.move(%s, %s)" % (DOWN, linenr))
self.log.append("self.move(%s, %s)" % (RIGHT, col))
self.treemanager.cursor_reset()
self.move(DOWN, linenr)
self.move(RIGHT, col)
k = self.get_random_key()
if k in [
"a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7"
]:
# for a few characters DELETE instead of INSERT
self.log.append("self.treemanager.key_delete()")
x = self.treemanager.key_delete()
else:
rk = self.get_random_key()
self.log.append("self.treemanager.key_normal(%s)" % rk)
x = self.treemanager.key_normal(rk)
if x == "eos":
continue
self.log.append("self.treemanager.undo_snapshot()")
self.treemanager.undo_snapshot()
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
t1 = TreeManager()
parser, lexer = self.lang.load()
t1.add_parser(parser, lexer, self.lang.name)
t1.import_file(self.program)
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
class FuzzyTester():
"""Runs test that randomly modify a given program and tests for exceptions.
If an exception occurs, or the modifications had unwanted side-effects, a
log is saved which can be used to create a stand-alone test."""
def __init__(self, filename):
_, ext = os.path.splitext(filename)
self.setlang(ext_to_lang[ext])
with open(filename) as f:
self.program = f.read()
self.log = []
self.filename = filename
def reset(self):
self.parser.reset()
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, self.lang.name)
def setlang(self, lang):
self.lang = lang_dict[lang]
self.parser, self.lexer = lang_dict[lang].load()
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, self.lang.name)
def move(self, direction, times):
for i in range(times):
self.treemanager.cursor_movement(direction)
def text_compare(self, original):
"""Compare the textual representation of two programs. Faster than
tree_compare, but less accurate."""
original = original.replace("\r", "").split("\n")
current = self.treemanager.export_as_text("/dev/null").replace(
"\r", "").split("\n")
for i in xrange(len(current)):
assert original[i] == current[i]
def next_node(self, node):
if node.children:
return node.children[0]
while node.right_sibling() is None:
node = node.parent
return node.right_sibling()
def tree_compare(self, node1, node2):
"""Given two root nodes, compares that both trees are equivalent"""
while True:
assert node1.symbol == node2.symbol
if node1.right:
assert node1.right.symbol == node2.right.symbol
if node1.next_term:
assert node1.next_term.symbol == node2.next_term.symbol
if isinstance(node1.symbol, MagicTerminal):
self.tree_compare(node1.symbol.ast, node2.symbol.ast)
if isinstance(node1, EOS) and isinstance(node2, EOS):
break
node1 = self.next_node(node1)
node2 = self.next_node(node2)
def get_random_key(self):
keys = list(
"abcdefghijklmnopqrstuvwxyz0123456789 \r:,.[]{}()!$%^&*()_+=")
return random.choice(keys)
def random_deletion(self):
"""Delete random characters within a program."""
print "Running random_deletion on {}".format(self.filename)
program = self.program
self.treemanager.import_file(program)
assert self.parser.last_status == True
self.text_compare(program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
random_lines = random_lines[:
20] # restrict to 20 lines to reduce runtime
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.treemanager.cursor_reset()
self.log.append("self.treemanager.cursor_reset()")
self.move(DOWN, linenr)
self.log.append("self.move(DOWN, %s)" % linenr)
self.move(RIGHT, col)
self.log.append("self.move(RIGHT, %s)" % col)
self.log.append("self.treemanager.key_delete()")
x = self.treemanager.key_delete()
if x == "eos":
continue
self.treemanager.undo_snapshot()
self.log.append("self.treemanager.undo_snapshot()")
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(program)
t1 = TreeManager()
parser, lexer = self.lang.load()
parser.init_ast()
t1.add_parser(parser, lexer, self.lang.name)
t1.set_font_test(7, 17)
t1.import_file(self.program)
assert self.parser.last_status == True
assert parser.last_status == True
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
def random_insertion(self):
"""Insert random characters at random locations within a program."""
print "Running random_insert on {}".format(self.filename)
self.reset()
self.treemanager.import_file(self.program)
assert self.parser.last_status == True
self.text_compare(self.program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.log.append("self.treemanager.cursor_reset()")
self.log.append("self.move(DOWN, %s)" % linenr)
self.log.append("self.move(RIGHT, %s)" % col)
self.treemanager.cursor_reset()
self.move(DOWN, linenr)
self.move(RIGHT, col)
k = self.get_random_key()
self.log.append("self.treemanager.key_normal(%s)" % repr(k))
x = self.treemanager.key_normal(k)
if x == "eos":
continue
self.log.append("self.treemanager.undo_snapshot()")
self.treemanager.undo_snapshot()
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
t1 = TreeManager()
parser, lexer = self.lang.load()
t1.add_parser(parser, lexer, self.lang.name)
t1.import_file(self.program)
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
def random_insertdelete(self):
"""Insert and delete random characters at random locations within a
program."""
print "Running random_insertdelete on {}".format(self.filename)
self.reset()
self.treemanager.import_file(self.program)
assert self.parser.last_status == True
self.text_compare(self.program)
line_count = len(self.treemanager.lines)
random_lines = range(line_count)
random.shuffle(random_lines)
random_lines = random_lines[:
20] # restrict to 20 lines to reduce runtime
start_version = self.treemanager.version
for linenr in random_lines:
cols = range(20)
random.shuffle(cols)
for col in cols:
self.log.append("self.treemanager.cursor_reset()")
self.log.append("self.move(%s, %s)" % (DOWN, linenr))
self.log.append("self.move(%s, %s)" % (RIGHT, col))
self.treemanager.cursor_reset()
self.move(DOWN, linenr)
self.move(RIGHT, col)
k = self.get_random_key()
if k in [
"a", "c", "e", "g", "i", "k", "m", "1", "3", "5", "7"
]:
# for a few characters DELETE instead of INSERT
self.log.append("self.treemanager.key_delete()")
x = self.treemanager.key_delete()
else:
rk = self.get_random_key()
self.log.append("self.treemanager.key_normal(%s)" % rk)
x = self.treemanager.key_normal(rk)
if x == "eos":
continue
self.log.append("self.treemanager.undo_snapshot()")
self.treemanager.undo_snapshot()
end_version = self.treemanager.version
broken = self.treemanager.export_as_text()
# undo all and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
# redo all and compare with broken
while self.treemanager.version < end_version:
self.treemanager.key_shift_ctrl_z()
self.text_compare(broken)
# undo again and compare with original
while self.treemanager.version > start_version:
self.treemanager.key_ctrl_z()
self.text_compare(self.program)
t1 = TreeManager()
parser, lexer = self.lang.load()
t1.add_parser(parser, lexer, self.lang.name)
t1.import_file(self.program)
self.tree_compare(self.parser.previous_version.parent,
parser.previous_version.parent)
def run(self):
try:
ft.random_deletion()
self.reset()
ft.random_insertion()
self.reset()
ft.random_insertdelete()
except Exception as e:
traceback.print_exc()
print "Written log to 'fuzzy.log'."
with open("fuzzy.log", "w") as f:
for l in self.log:
f.write(l)
f.write("\n")
else:
print "Passed."
class FuzzyLboxStats:
def __init__(self, main, sub):
parser, lexer = main.load()
self.lexer = lexer
self.parser = parser
self.ast = parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(parser, lexer, main.name)
parser.setup_autolbox(main.name)
self.sub = sub
self.inserted = 0
def load_main(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.treemanager.import_file(content)
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
def load_expr(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.replexprs = self.find_expressions(content, self.sub_repl_str)
def set_replace(self, main, sub):
self.main_repl_str = main
self.sub_repl_str = sub
def find_nonterms_by_name(self, tm, name):
l = []
bos = tm.get_bos()
eos = tm.get_eos()
node = bos.right_sibling()
while node is not eos:
if node.symbol.name == name:
l.append(node)
if node.children:
node = node.children[0]
continue
node = next_node(node)
return l
def find_expressions(self, program, expr):
parser, lexer = self.sub.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, self.sub.name)
treemanager.import_file(program)
# find all sub expressions
l = self.find_nonterms_by_name(treemanager, expr)
return [subtree_to_text(st).rstrip() for st in l]
def insert_python_expression(self, expr):
for c in expr:
self.treemanager.key_normal(c)
def delete_expr(self, expr):
# find first term and last term
# select + delete
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[0]
else:
node = next_node(node)
first = node
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[-1]
else:
node = prev_node(node)
last = node
if first.deleted or last.deleted:
return None
self.treemanager.select_nodes(first, last)
deleted = self.treemanager.copySelection()
self.treemanager.deleteSelection()
return deleted
def run(self):
assert len(self.treemanager.parsers) == 1
print self.main_repl_str, len(
[subtree_to_text(x) for x in self.mainexprs])
print self.sub_repl_str, len(self.replexprs)
random.shuffle(self.mainexprs)
for e in self.mainexprs[:10]:
if e.get_root() is None:
continue
deleted = self.delete_expr(e)
before = len(self.treemanager.parsers)
if deleted:
choice = random.choice(self.replexprs)
print " Replacing '{}' with '{}':".format(
truncate(deleted), truncate(choice))
self.insert_python_expression(choice)
valid = self.parser.last_status
if before == len(self.treemanager.parsers):
result = "No box inserted"
else:
result = "Box inserted"
self.inserted += 1
print " => {} ({})".format(result, valid)
else:
print "Replacing '{}' with '{}':\n => Already deleted".format(
truncate(subtree_to_text(e)), truncate(choice))
print("Boxes inserted: {}/{}".format(self.inserted, 10))