def test_switch_grammar_unicode_terminal(self): """ This test checks that a parse tree built with a grammar containing only ascii characters can be reconstructed with a grammar that has unicode rules (or vice versa). The original bug assigned ANON terminals to unicode keywords, which offsets the ANON terminal count in the unicode grammar and causes subsequent identical ANON tokens (e.g., `+=`) to mis-match between the two grammars. """ g1 = """ start: (NL | stmt)* stmt: "keyword" var op var !op: ("+=" | "-=" | "*=" | "/=") var: WORD NL: /(\\r?\\n)+\s*/ """ + common g2 = """ start: (NL | stmt)* stmt: "குறிப்பு" var op var !op: ("+=" | "-=" | "*=" | "/=") var: WORD NL: /(\\r?\\n)+\s*/ """ + common code = """ keyword x += y """ l1 = Lark(g1, parser='lalr', maybe_placeholders=False) l2 = Lark(g2, parser='lalr', maybe_placeholders=False) r = Reconstructor(l2) tree = l1.parse(code) code2 = r.reconstruct(tree) assert l2.parse(code2) == tree
class Matcher(object): def __init__(self, tree, parser, matcher): assert matcher is not None assert tree is not None assert parser is not None self.matcher = matcher self.tree = tree self.parser = parser self._reconstructor = Reconstructor(parser) def match(self, array): if not isinstance(array, set): array = set(array) return self.matcher(array) def pretty(self): return self.tree.pretty() def query_str(self): self._reconstructor.reconstruct(self.tree)
def parse(self, env: Environment, string: str) -> str: """Parse input, substitute variable and return the result.""" try: tree = self.parser.parse(string) transformer = SubstitutionTransformer(env) tree = transformer.transform(tree) reconstructor = Reconstructor(self.parser) return reconstructor.reconstruct(tree) except (UnexpectedCharacters, UnexpectedToken) as e: raise ShellException('[Substitution]Unexpected characters at position %s' % e.pos_in_stream) except LarkError: raise ShellException('[Substitution]Parse error')
def replace_var_in_expression(expression, old_var, new_var, parser=None, matching_terminal_names=('ID')): """ Replaces all occurrences of old_var in expression with new_var. :param matching_terminal_names: Token names according to the grammar taken into account for replacement. :type matching_terminal_names: tuple (of strings) :param expression: An expression in the grammar used by parser. :type expression: str :param old_var: Old variable :type old_var: str :param new_var: New variable :type new_var: str :param parser: Lark parser :type parser: Lark (if not set default parser will be used.) :return: Expression with replaced variable. :rtype: str """ if parser is None: parser = get_parser_instance() tree = parser.parse(expression) recons = Reconstructor(parser) for node in tree.iter_subtrees(): # type: Tree for child in node.children: if isinstance(child, Token): pass if isinstance(child, Token) and child.type in matching_terminal_names: if child.value == old_var: node.set(data=node.data, children=[Token(child.type, new_var)]) return recons.reconstruct(tree)
class XLMInterpreter: def __init__(self, xlm_wrapper): self.xlm_wrapper = xlm_wrapper self.cell_addr_regex_str = r"((?P<sheetname>[^\s]+?|'.+?')!)?\$?(?P<column>[a-zA-Z]+)\$?(?P<row>\d+)" self.cell_addr_regex = re.compile(self.cell_addr_regex_str) macro_grammar = open('xlm-macro.lark', 'r', encoding='utf_8').read() self.xlm_parser = Lark(macro_grammar, parser='lalr') self.defined_names = self.xlm_wrapper.get_defined_names() self.tree_reconstructor = Reconstructor(self.xlm_parser) def is_float(self, text): try: float(text) return True except ValueError: return False def is_int(self, text): try: int(text) return True except ValueError: return False except TypeError: return False def get_formula_cell(self, macrosheet, col, row): result_cell = None not_found = False row = int(row) current_row = row current_addr = col + str(current_row) while current_addr not in macrosheet.cells or \ macrosheet.cells[current_addr].formula is None: if (current_row - row) < 50: current_row += 1 else: not_found = True break current_addr = col + str(current_row) if not_found is False: result_cell = macrosheet.cells[current_addr] return result_cell def get_argument_length(self, arglist_node): result = None if arglist_node.data == 'arglist': result = len(arglist_node.children) return result def get_cell(self, current_cell, cell_parse_tree): res_sheet = res_col = res_row = None if type(cell_parse_tree) is Token: names = self.xlm_wrapper.get_defined_names() label = cell_parse_tree.value if label in names: res_sheet, res_col, res_row = Cell.parse_cell_addr( names[cell_parse_tree]) else: cell = cell_parse_tree.children[0] if cell.data == 'absolute_cell': res_sheet, res_col, res_row = Cell.parse_cell_addr( cell.children[0]) if res_sheet is None: res_sheet = current_cell.sheet.name elif cell.data == 'relative_cell': first_child = cell.children[0] second_child = cell.children[1] res_sheet = current_cell.sheet.name res_col = Cell.convert_to_column_index(current_cell.column) res_row = int(current_cell.row) if first_child == 'R' and self.is_int(second_child): res_row = res_row + int(second_child) if len(cell.children) == 4: res_col = res_col + int(cell.children[4]) elif second_child == 'c': res_col = res_col + int(cell.children[2]) res_row = str(res_row) res_col = Cell.convert_to_column_name(res_col) else: raise Exception('Cell addresss, Syntax Error') return res_sheet, res_col, res_row def set_cell(self, sheet_name, col, row, text): sheets = self.xlm_wrapper.get_macrosheets() if sheet_name in sheets: sheet = sheets[sheet_name] addr = col + str(row) if addr not in sheet.cells: new_cell = Cell() new_cell.column = col new_cell.row = row new_cell.sheet = sheet sheet.cells[addr] = new_cell cell = sheet.cells[addr] if text.startswith('='): cell.formula = text else: cell.value = text def evaluate_parse_tree(self, current_cell, parse_tree_root, interactive=True): next_cell = None status = EvalStatus.NotImplemented text = None return_val = None if type(parse_tree_root) is Token: text = parse_tree_root.value status = EvalStatus.FullEvaluation return_val = text elif parse_tree_root.data == 'function_call': function_name = parse_tree_root.children[0] function_arguments = parse_tree_root.children[1] size = self.get_argument_length(function_arguments) if function_name == 'RUN': if size == 1: next_sheet, next_col, next_row = self.get_cell( current_cell, function_arguments.children[0].children[0]) if next_sheet is not None and next_sheet in self.xlm_wrapper.get_macrosheets( ): next_cell = self.get_formula_cell( self.xlm_wrapper.get_macrosheets()[next_sheet], next_col, next_row) text = 'RUN({}!{}{})'.format(next_sheet, next_col, next_row) status = EvalStatus.FullEvaluation else: status = EvalStatus.Error text = self.tree_reconstructor.reconstruct( parse_tree_root) return_val = 0 elif size == 2: text = 'RUN(reference, step)' status = EvalStatus.NotImplemented else: text = 'RUN() is incorrect' status = EvalStatus.Error elif function_name == 'CHAR': next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, function_arguments.children[0], interactive) if status == EvalStatus.FullEvaluation: text = chr(int(text)) cell = self.get_formula_cell(current_cell.sheet, current_cell.column, current_cell.row) cell.value = text return_val = text elif function_name == 'FORMULA': first_arg = function_arguments.children[0] next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, first_arg, interactive) second_arg = function_arguments.children[1].children[0] dst_sheet, dst_col, dst_row = self.get_cell( current_cell, second_arg) if status == EvalStatus.FullEvaluation: self.set_cell(dst_sheet, dst_col, dst_row, text) text = "FORMULA({},{})".format( text, '{}!{}{}'.format(dst_sheet, dst_col, dst_row)) return_val = 0 elif function_name == 'CALL': arguments = [] status = EvalStatus.FullEvaluation for argument in function_arguments.children: next_cell, tmp_status, return_val, text = self.evaluate_parse_tree( current_cell, argument, interactive) if tmp_status == EvalStatus.FullEvaluation: if text is not None: arguments.append(text) else: arguments.append(' ') else: status = tmp_status arguments.append('not evaluated') text = 'CALL({})'.format(','.join(arguments)) return_val = 0 elif function_name in ('HALT', 'CLOSE'): next_row = None next_col = None next_sheet = None text = self.tree_reconstructor.reconstruct(parse_tree_root) status = EvalStatus.End elif function_name == 'GOTO': next_sheet, next_col, next_row = self.get_cell( current_cell, function_arguments.children[0].children[0]) if next_sheet is not None and next_sheet in self.xlm_wrapper.get_macrosheets( ): next_cell = self.get_formula_cell( self.xlm_wrapper.get_macrosheets()[next_sheet], next_col, next_row) status = EvalStatus.FullEvaluation else: status = EvalStatus.Error text = self.tree_reconstructor.reconstruct(parse_tree_root) elif function_name.lower() in self.defined_names: cell_text = self.defined_names[function_name.lower()] next_sheet, next_col, next_row = self.parse_cell_address( cell_text) text = 'Label ' + function_name status = EvalStatus.FullEvaluation elif function_name == 'ERROR': text = 'ERROR' status = EvalStatus.FullEvaluation elif function_name == 'IF': if size == 3: second_arg = function_arguments.children[1] next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, second_arg, interactive) if status == EvalStatus.FullEvaluation: third_arg = function_arguments.children[2] status = EvalStatus.PartialEvaluation else: status = EvalStatus.FullEvaluation text = self.tree_reconstructor.reconstruct(parse_tree_root) elif function_name == 'NOW': text = datetime.datetime.now() status = EvalStatus.FullEvaluation elif function_name == 'DAY': first_arg = function_arguments.children[0] next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, first_arg, interactive) if status == EvalStatus.FullEvaluation: if type(text) is datetime.datetime: text = str(text.day) status = EvalStatus.FullEvaluation elif self.is_float(text): text = 'DAY(Serial Date)' status = EvalStatus.NotImplemented else: # args_str ='' # for argument in function_arguments.children: # next_cell, status, return_val, text = self.evaluate_parse_tree(current_cell, argument, interactive) # args_str += str(return_val) +',' # args_str.strip(',') # text = '{}({})'.format(function_name, args_str) text = self.tree_reconstructor.reconstruct(parse_tree_root) status = EvalStatus.NotImplemented elif parse_tree_root.data == 'method_call': text = self.tree_reconstructor.reconstruct(parse_tree_root) status = EvalStatus.NotImplemented elif parse_tree_root.data == 'cell': sheet_name, col, row = self.get_cell(current_cell, parse_tree_root) cell_addr = col + str(row) sheet = self.xlm_wrapper.get_macrosheets()[sheet_name] missing = True if cell_addr not in sheet.cells or sheet.cells[ cell_addr].value is None: if interactive: self.interactive_shell( current_cell, '{} is not populated, what should be its value?'. format(cell_addr)) if cell_addr in sheet.cells: cell = sheet.cells[cell_addr] if cell.value is not None: text = cell.value status = EvalStatus.FullEvaluation return_val = text missing = False else: text = "{}".format(cell_addr) else: text = "{}".format(cell_addr) elif parse_tree_root.data == 'binary_expression': left_arg = parse_tree_root.children[0] next_cell, l_status, return_val, text_left = self.evaluate_parse_tree( current_cell, left_arg, interactive) operator = str(parse_tree_root.children[1].children[0]) right_arg = parse_tree_root.children[2] next_cell, r_status, return_val, text_right = self.evaluate_parse_tree( current_cell, right_arg, interactive) if l_status == EvalStatus.FullEvaluation and r_status == EvalStatus.FullEvaluation: status = EvalStatus.FullEvaluation if operator == '&': text = text_left + text_right elif self.is_int(text_left) and self.is_int(text_right): if operator == '-': text = str(int(text_left) - int(text_right)) elif operator == '+': text = str(int(text_left) + int(text_right)) elif operator == '*': text = str(int(text_left) * int(text_right)) else: text = 'Operator ' + operator status = EvalStatus.NotImplemented else: text = self.tree_reconstructor.reconstruct(parse_tree_root) status = EvalStatus.PartialEvaluation else: status = EvalStatus.PartialEvaluation text = '{}{}{}'.format(text_left, operator, text_right) return_val = text else: status = EvalStatus.FullEvaluation for child_node in parse_tree_root.children: if child_node is not None: next_cell, tmp_status, return_val, text = self.evaluate_parse_tree( current_cell, child_node, interactive) if tmp_status != EvalStatus.FullEvaluation: status = tmp_status return next_cell, status, return_val, text def interactive_shell(self, current_cell, message): print('\nProcess Interruption:') print('CELL:{:10}{}'.format(current_cell.get_local_address(), current_cell.formula)) print(message) print('Enter XLM macro:') print('Tip: CLOSE() or HALT() to exist') while True: line = input() line = '=' + line.strip().strip('"') if line: try: parse_tree = self.xlm_parser.parse(line) next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, parse_tree, interactive=False) print(return_val) if status == EvalStatus.End: break except ParseError: print("Invalid XLM macro") else: break def deobfuscate_macro(self, interactive): result = [] auto_open_labels = self.xlm_wrapper.get_defined_name('auto_open', full_match=False) if auto_open_labels is not None and len(auto_open_labels) > 0: macros = self.xlm_wrapper.get_macrosheets() print('[Starting Deobfuscation]') for auto_open_label in auto_open_labels: sheet_name, col, row = Cell.parse_cell_addr(auto_open_label[1]) current_cell = self.get_formula_cell(macros[sheet_name], col, row) self.branches = [] while current_cell is not None: parse_tree = self.xlm_parser.parse(current_cell.formula) next_cell, status, return_val, text = self.evaluate_parse_tree( current_cell, parse_tree, interactive) if return_val is not None: current_cell.value = str(return_val) if next_cell is None and status != EvalStatus.Error: next_cell = self.get_formula_cell( current_cell.sheet, current_cell.column, str(int(current_cell.row) + 1)) yield (current_cell, status, text) if next_cell is not None: current_cell = next_cell else: break