class FuzzyLboxStats: def __init__(self, main, sub): parser, lexer = main.load() self.lexer = lexer self.parser = parser self.ast = parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(parser, lexer, main.name) self.treemanager.option_autolbox_insert = True self.langname = main.name parser.setup_autolbox(main.name, lexer) self.sub = sub self.inserted = 0 self.log = [] def load_main(self, filename): self.filename = filename f = open(filename, "r") self.content = f.read() f.close() self.content.replace("\n", "\r") self.treemanager.import_file(self.content) self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) self.minver = self.treemanager.version def reset(self): self.parser.reset() self.ast = self.parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(self.parser, self.lexer, self.langname) self.treemanager.import_file(self.content) self.treemanager.option_autolbox_insert = True self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) def load_expr(self, filename): f = open(filename, "r") content = f.read() f.close() self.replexprs = self.find_expressions(content, self.sub_repl_str) def load_expr_from_json(self, filename): import json with open(filename) as f: self.replexprs = json.load(f) def set_replace(self, main, sub): self.main_repl_str = main self.sub_repl_str = sub def find_nonterms_by_name(self, tm, name): l = [] bos = tm.get_bos() eos = tm.get_eos() node = bos.right_sibling() while node is not eos: if validnonterm(node, name): l.append(node) node = next_node(node) continue if node.children: node = node.children[0] else: node = next_node(node) return l def find_expressions(self, program, expr): parser, lexer = self.sub.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, self.sub.name) treemanager.import_file(program) # find all sub expressions l = self.find_nonterms_by_name(treemanager, expr) return [subtree_to_text(st).rstrip() for st in l] def insert_python_expression(self, expr): for c in expr: self.treemanager.key_normal(c) def delete_expr(self, expr): # find first term and last term # select + delete node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[0] else: node = next_node(node) first = node if isinstance(first, MultiTextNode): first = first.children[0] node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[-1] else: node = prev_node(node) while node.lookup == "<ws>" or node.lookup == "<return>": node = node.prev_term last = node if isinstance(last, MultiTextNode): last = last.children[-1] if first.deleted or last.deleted: return None self.treemanager.select_nodes(first, last) deleted = self.treemanager.copySelection() self.treemanager.deleteSelection() return deleted def multi_len(self, autos): r = [] for start, end, _, _ in autos: l = 0 while start is not end: l += len(start.symbol.name) start = start.next_term r.append(l) return r def run(self, main_samples=None, sub_samples=None): assert len(self.treemanager.parsers) == 1 ops = self.main_repl_str, len( [subtree_to_text(x) for x in self.mainexprs]) preversion = self.treemanager.version inserted_error = 0 inserted_valid = 0 noinsert_error = 0 noinsert_valid = 0 noinsert_multi = 0 # pick random exprs from main if not main_samples: samplesize = 10 if len(self.mainexprs) < 10: samplesize = len(self.mainexprs) sample = random.sample(range(len(self.mainexprs)), samplesize) # store this for repeatability exprchoices = [self.mainexprs[i] for i in sample] self.main_samples = sample else: self.main_samples = main_samples exprchoices = [self.mainexprs[i] for i in main_samples] if not sub_samples: # pick random exprs from sub sample = random.sample(range(len(self.replexprs)), len(exprchoices)) replchoices = [self.replexprs[i] for i in sample] self.sub_samples = sample else: self.sub_samples = sub_samples replchoices = [self.replexprs[i] for i in sub_samples] for i in range(len(exprchoices)): e = exprchoices[i] if e.get_root() is None: continue before = len(self.treemanager.parsers) deleted = self.delete_expr(e) mboxes = None ilength = None if deleted: choice = replchoices[i].strip() choice_len = len(choice) if debug: print " Replacing '{}' with '{}':".format( repr(truncate(deleted)), repr(choice)) start = time.time() cursor = self.treemanager.cursor if cursor.node.lookup != "<ws>": # Insert a leading space in situations like `return(x)`. choice = " " + choice if cursor.node.symbol.name[ cursor. pos:] == "" and cursor.node.next_term.lookup != "<ws>": # Insert trailing space if there is none choice = choice + " " self.insert_python_expression(choice) itime = time.time() - start valid = self.parser.last_status if before == len(self.treemanager.parsers): if len( self.parser.error_nodes ) > 0 and self.parser.error_nodes[0].autobox and len( self.parser.error_nodes[0].autobox) > 1: noinsert_multi += 1 result = "No box inserted (Multi)" outcome = "multi" mboxes = self.multi_len( self.parser.error_nodes[0].autobox) elif valid: noinsert_valid += 1 result = "No box inserted (Valid)" outcome = "valid" else: noinsert_error += 1 result = "No box inserted (Error)" outcome = "error" else: result = "Box inserted" innervalid = self.treemanager.parsers[-1][0].last_status self.inserted += 1 recent_box = self.treemanager.parsers[-1][ 0].previous_version.parent lbox_len = self.lbox_length(recent_box) ilength = (lbox_len, choice_len) if valid and innervalid: inserted_valid += 1 outcome = "ok" else: inserted_error += 1 outcome = "inerr" if debug: print " => {} ({})".format(result, valid) nlboxes = len(self.treemanager.parsers) self.log.append([ self.filename, repr(deleted), repr(choice), outcome, itime, mboxes, ilength, nlboxes ]) else: if debug: print "Replacing '{}' with '{}':\n => Already deleted".format( truncate(subtree_to_text(e)), truncate(choice)) self.undo(self.minver) exprchoices = [self.mainexprs[i] for i in self.main_samples] if debug: print("Boxes inserted: {}/{}".format(self.inserted, ops)) print("Valid insertions:", inserted_valid) print("Invalid insertions:", inserted_error) print("No insertion (valid):", noinsert_valid) print("No insertion (error):", noinsert_error) print("No insertion (multi):", noinsert_multi) return (inserted_valid, inserted_error, noinsert_valid, noinsert_error, noinsert_multi) def undo(self, version): # reset everything self.reset() return while self.treemanager.version != version: before = self.treemanager.version self.treemanager.version -= 1 self.treemanager.recover_version("undo", self.treemanager.version + 1) self.treemanager.cursor.load(self.treemanager.version, self.treemanager.lines) if before == self.treemanager.version: exit("Error") def lbox_length(self, root): l = [] node = root.children[0] eos = root.children[-1] while node is not eos: if not node.deleted: l.append(node.symbol.name) node = node.next_term s = "".join(l).strip() return len(s)
class FuzzyLboxStats: def __init__(self, main, sub): parser, lexer = main.load() self.lexer = lexer self.parser = parser self.ast = parser.previous_version self.treemanager = TreeManager() self.treemanager.add_parser(parser, lexer, main.name) parser.setup_autolbox(main.name) self.sub = sub self.inserted = 0 def load_main(self, filename): f = open(filename, "r") content = f.read() f.close() self.treemanager.import_file(content) self.mainexprs = self.find_nonterms_by_name(self.treemanager, self.main_repl_str) def load_expr(self, filename): f = open(filename, "r") content = f.read() f.close() self.replexprs = self.find_expressions(content, self.sub_repl_str) def set_replace(self, main, sub): self.main_repl_str = main self.sub_repl_str = sub def find_nonterms_by_name(self, tm, name): l = [] bos = tm.get_bos() eos = tm.get_eos() node = bos.right_sibling() while node is not eos: if node.symbol.name == name: l.append(node) if node.children: node = node.children[0] continue node = next_node(node) return l def find_expressions(self, program, expr): parser, lexer = self.sub.load() treemanager = TreeManager() treemanager.add_parser(parser, lexer, self.sub.name) treemanager.import_file(program) # find all sub expressions l = self.find_nonterms_by_name(treemanager, expr) return [subtree_to_text(st).rstrip() for st in l] def insert_python_expression(self, expr): for c in expr: self.treemanager.key_normal(c) def delete_expr(self, expr): # find first term and last term # select + delete node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[0] else: node = next_node(node) first = node node = expr while type(node.symbol) is Nonterminal: if node.children: node = node.children[-1] else: node = prev_node(node) last = node if first.deleted or last.deleted: return None self.treemanager.select_nodes(first, last) deleted = self.treemanager.copySelection() self.treemanager.deleteSelection() return deleted def run(self): assert len(self.treemanager.parsers) == 1 print self.main_repl_str, len( [subtree_to_text(x) for x in self.mainexprs]) print self.sub_repl_str, len(self.replexprs) random.shuffle(self.mainexprs) for e in self.mainexprs[:10]: if e.get_root() is None: continue deleted = self.delete_expr(e) before = len(self.treemanager.parsers) if deleted: choice = random.choice(self.replexprs) print " Replacing '{}' with '{}':".format( truncate(deleted), truncate(choice)) self.insert_python_expression(choice) valid = self.parser.last_status if before == len(self.treemanager.parsers): result = "No box inserted" else: result = "Box inserted" self.inserted += 1 print " => {} ({})".format(result, valid) else: print "Replacing '{}' with '{}':\n => Already deleted".format( truncate(subtree_to_text(e)), truncate(choice)) print("Boxes inserted: {}/{}".format(self.inserted, 10))