class FuzzyLboxStats:
def __init__(self, main, sub):
parser, lexer = main.load()
self.lexer = lexer
self.parser = parser
self.ast = parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(parser, lexer, main.name)
self.treemanager.option_autolbox_insert = True
self.langname = main.name
parser.setup_autolbox(main.name, lexer)
self.sub = sub
self.inserted = 0
self.log = []
def load_main(self, filename):
self.filename = filename
f = open(filename, "r")
self.content = f.read()
f.close()
self.content.replace("\n", "\r")
self.treemanager.import_file(self.content)
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
self.minver = self.treemanager.version
def reset(self):
self.parser.reset()
self.ast = self.parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(self.parser, self.lexer, self.langname)
self.treemanager.import_file(self.content)
self.treemanager.option_autolbox_insert = True
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
def load_expr(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.replexprs = self.find_expressions(content, self.sub_repl_str)
def load_expr_from_json(self, filename):
import json
with open(filename) as f:
self.replexprs = json.load(f)
def set_replace(self, main, sub):
self.main_repl_str = main
self.sub_repl_str = sub
def find_nonterms_by_name(self, tm, name):
l = []
bos = tm.get_bos()
eos = tm.get_eos()
node = bos.right_sibling()
while node is not eos:
if validnonterm(node, name):
l.append(node)
node = next_node(node)
continue
if node.children:
node = node.children[0]
else:
node = next_node(node)
return l
def find_expressions(self, program, expr):
parser, lexer = self.sub.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, self.sub.name)
treemanager.import_file(program)
# find all sub expressions
l = self.find_nonterms_by_name(treemanager, expr)
return [subtree_to_text(st).rstrip() for st in l]
def insert_python_expression(self, expr):
for c in expr:
self.treemanager.key_normal(c)
def delete_expr(self, expr):
# find first term and last term
# select + delete
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[0]
else:
node = next_node(node)
first = node
if isinstance(first, MultiTextNode):
first = first.children[0]
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[-1]
else:
node = prev_node(node)
while node.lookup == "<ws>" or node.lookup == "<return>":
node = node.prev_term
last = node
if isinstance(last, MultiTextNode):
last = last.children[-1]
if first.deleted or last.deleted:
return None
self.treemanager.select_nodes(first, last)
deleted = self.treemanager.copySelection()
self.treemanager.deleteSelection()
return deleted
def multi_len(self, autos):
r = []
for start, end, _, _ in autos:
l = 0
while start is not end:
l += len(start.symbol.name)
start = start.next_term
r.append(l)
return r
def run(self, main_samples=None, sub_samples=None):
assert len(self.treemanager.parsers) == 1
ops = self.main_repl_str, len(
[subtree_to_text(x) for x in self.mainexprs])
preversion = self.treemanager.version
inserted_error = 0
inserted_valid = 0
noinsert_error = 0
noinsert_valid = 0
noinsert_multi = 0
# pick random exprs from main
if not main_samples:
samplesize = 10
if len(self.mainexprs) < 10:
samplesize = len(self.mainexprs)
sample = random.sample(range(len(self.mainexprs)),
samplesize) # store this for repeatability
exprchoices = [self.mainexprs[i] for i in sample]
self.main_samples = sample
else:
self.main_samples = main_samples
exprchoices = [self.mainexprs[i] for i in main_samples]
if not sub_samples:
# pick random exprs from sub
sample = random.sample(range(len(self.replexprs)),
len(exprchoices))
replchoices = [self.replexprs[i] for i in sample]
self.sub_samples = sample
else:
self.sub_samples = sub_samples
replchoices = [self.replexprs[i] for i in sub_samples]
for i in range(len(exprchoices)):
e = exprchoices[i]
if e.get_root() is None:
continue
before = len(self.treemanager.parsers)
deleted = self.delete_expr(e)
mboxes = None
ilength = None
if deleted:
choice = replchoices[i].strip()
choice_len = len(choice)
if debug:
print " Replacing '{}' with '{}':".format(
repr(truncate(deleted)), repr(choice))
start = time.time()
cursor = self.treemanager.cursor
if cursor.node.lookup != "<ws>":
# Insert a leading space in situations like `return(x)`.
choice = " " + choice
if cursor.node.symbol.name[
cursor.
pos:] == "" and cursor.node.next_term.lookup != "<ws>":
# Insert trailing space if there is none
choice = choice + " "
self.insert_python_expression(choice)
itime = time.time() - start
valid = self.parser.last_status
if before == len(self.treemanager.parsers):
if len(
self.parser.error_nodes
) > 0 and self.parser.error_nodes[0].autobox and len(
self.parser.error_nodes[0].autobox) > 1:
noinsert_multi += 1
result = "No box inserted (Multi)"
outcome = "multi"
mboxes = self.multi_len(
self.parser.error_nodes[0].autobox)
elif valid:
noinsert_valid += 1
result = "No box inserted (Valid)"
outcome = "valid"
else:
noinsert_error += 1
result = "No box inserted (Error)"
outcome = "error"
else:
result = "Box inserted"
innervalid = self.treemanager.parsers[-1][0].last_status
self.inserted += 1
recent_box = self.treemanager.parsers[-1][
0].previous_version.parent
lbox_len = self.lbox_length(recent_box)
ilength = (lbox_len, choice_len)
if valid and innervalid:
inserted_valid += 1
outcome = "ok"
else:
inserted_error += 1
outcome = "inerr"
if debug: print " => {} ({})".format(result, valid)
nlboxes = len(self.treemanager.parsers)
self.log.append([
self.filename,
repr(deleted),
repr(choice), outcome, itime, mboxes, ilength, nlboxes
])
else:
if debug:
print "Replacing '{}' with '{}':\n => Already deleted".format(
truncate(subtree_to_text(e)), truncate(choice))
self.undo(self.minver)
exprchoices = [self.mainexprs[i] for i in self.main_samples]
if debug:
print("Boxes inserted: {}/{}".format(self.inserted, ops))
print("Valid insertions:", inserted_valid)
print("Invalid insertions:", inserted_error)
print("No insertion (valid):", noinsert_valid)
print("No insertion (error):", noinsert_error)
print("No insertion (multi):", noinsert_multi)
return (inserted_valid, inserted_error, noinsert_valid, noinsert_error,
noinsert_multi)
def undo(self, version):
# reset everything
self.reset()
return
while self.treemanager.version != version:
before = self.treemanager.version
self.treemanager.version -= 1
self.treemanager.recover_version("undo",
self.treemanager.version + 1)
self.treemanager.cursor.load(self.treemanager.version,
self.treemanager.lines)
if before == self.treemanager.version:
exit("Error")
def lbox_length(self, root):
l = []
node = root.children[0]
eos = root.children[-1]
while node is not eos:
if not node.deleted:
l.append(node.symbol.name)
node = node.next_term
s = "".join(l).strip()
return len(s)
class FuzzyLboxStats:
def __init__(self, main, sub):
parser, lexer = main.load()
self.lexer = lexer
self.parser = parser
self.ast = parser.previous_version
self.treemanager = TreeManager()
self.treemanager.add_parser(parser, lexer, main.name)
parser.setup_autolbox(main.name)
self.sub = sub
self.inserted = 0
def load_main(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.treemanager.import_file(content)
self.mainexprs = self.find_nonterms_by_name(self.treemanager,
self.main_repl_str)
def load_expr(self, filename):
f = open(filename, "r")
content = f.read()
f.close()
self.replexprs = self.find_expressions(content, self.sub_repl_str)
def set_replace(self, main, sub):
self.main_repl_str = main
self.sub_repl_str = sub
def find_nonterms_by_name(self, tm, name):
l = []
bos = tm.get_bos()
eos = tm.get_eos()
node = bos.right_sibling()
while node is not eos:
if node.symbol.name == name:
l.append(node)
if node.children:
node = node.children[0]
continue
node = next_node(node)
return l
def find_expressions(self, program, expr):
parser, lexer = self.sub.load()
treemanager = TreeManager()
treemanager.add_parser(parser, lexer, self.sub.name)
treemanager.import_file(program)
# find all sub expressions
l = self.find_nonterms_by_name(treemanager, expr)
return [subtree_to_text(st).rstrip() for st in l]
def insert_python_expression(self, expr):
for c in expr:
self.treemanager.key_normal(c)
def delete_expr(self, expr):
# find first term and last term
# select + delete
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[0]
else:
node = next_node(node)
first = node
node = expr
while type(node.symbol) is Nonterminal:
if node.children:
node = node.children[-1]
else:
node = prev_node(node)
last = node
if first.deleted or last.deleted:
return None
self.treemanager.select_nodes(first, last)
deleted = self.treemanager.copySelection()
self.treemanager.deleteSelection()
return deleted
def run(self):
assert len(self.treemanager.parsers) == 1
print self.main_repl_str, len(
[subtree_to_text(x) for x in self.mainexprs])
print self.sub_repl_str, len(self.replexprs)
random.shuffle(self.mainexprs)
for e in self.mainexprs[:10]:
if e.get_root() is None:
continue
deleted = self.delete_expr(e)
before = len(self.treemanager.parsers)
if deleted:
choice = random.choice(self.replexprs)
print " Replacing '{}' with '{}':".format(
truncate(deleted), truncate(choice))
self.insert_python_expression(choice)
valid = self.parser.last_status
if before == len(self.treemanager.parsers):
result = "No box inserted"
else:
result = "Box inserted"
self.inserted += 1
print " => {} ({})".format(result, valid)
else:
print "Replacing '{}' with '{}':\n => Already deleted".format(
truncate(subtree_to_text(e)), truncate(choice))
print("Boxes inserted: {}/{}".format(self.inserted, 10))
