def test_comment(self): txt = '<!-- In the interest of restricting article length, please limit this section to two or three short ' \ 'paragraphs and add any substantial information to the main Issues in anarchism article. Thank you. ' \ '--> ' parser = Parser() ast = parser.parse(txt, Grammar.comment) print(ast, Compiler().render(ast))
def test_link(self): txt = '[[File:Nearest_stars_rotating_red-green.gif|alt=Rotating 3D image of the nearest stars|thumb|Animated 3D map of the nearest stars, centered on the Sun. {{3d glasses|color=red green}}]]' txt2 = '[[File:William Shea.jpg|thumb|upright|[[William Shea]] was instrumental in returning [[National League|National League baseball| [[asd|{{asd}}]]]] to [[New York City]] after five years of absence.]]' txt3 = '[[asd]]' parser = Parser() ast = parser.parse(txt2, Grammar.link) print(ast) return ast
def test_headings(self): txt = '==asd==' txt3 = '===asd===' txt4 = '====asd====' txt5 = '=====asd=====' txt6 = '======asd======' parser = Parser() ast = parser.parse(txt, expression=Grammar.headings) print(ast) return ast
def shell(fn, text): # Generate tokens lexer = Lexer(fn, text) tokens, exception = lexer.make_tokens() if exception: return None, exception # Generate AST parser = Parser(tokens) ast = parser.parse() return ast.node, ast.error
def test_parse(self, name='wikitext'): with (DATA_FOLDER / name).open(encoding="utf8") as f: text = f.read() t0 = time.time() # lexer = Lexer() # tokens = lexer.tokenize(text) parser = Parser() ast = parser.parse(text) t1 = time.time() # print(ast) print('Ast built in: ', t1 - t0) return ast
def test_template(self): parser = Parser() ast = parser.parse('{{asd}}', Grammar.template) print(ast) return ast
class Interpreter: """ Math expressions interpreter Supported types: Numbers(int or floats) Complex Numbers(using i) Matrices Variables Functions Variables and functions could be defined usign assignment operator Also, simple equations of degree 0-2 is supported. Every term should be in correct form: [coefficient][*][variable][^degree] Predefined math functions: sin, cos, tan, log, abs, sqrt, exp Predefined matrix functions: inv, transp Predefined special commands: vars, funcs, plot, linreg """ def __init__(self): self._variables = self._init_predefined_variables() self._functions = self._init_predefined_functions() self._parser = Parser() self._tokenizer = Tokenizer() def _init_predefined_variables(self): variables = {} for var_name, var_val in DEFINED_VARS.items(): variables[var_name] = Variable(var_name, var_val) return variables def _init_predefined_functions(self): functions = {} for func_name, func in DEFINED_FUNCS.items(): func_obj = SpecialNumericFunction(func_name, func) functions[func_name] = func_obj functions["inv"] = MatrixInversionFunc("inv") functions["transp"] = MatrixTransposeFunc("transp") return functions def read_eval_print_loop(self) -> None: """ Infinite REP loop which stops after key interrupt """ while True: try: input_string = input(">") except (EOFError, KeyboardInterrupt): break self.eval_print_string(input_string) def read_eval_print_file(self, filename: str) -> None: file = open(filename, "r") for line in file: self.eval_print_string(line) def eval_print_string(self, input_string: str) -> None: try: if input_string: output_string = self.eval_string(input_string) if output_string is not None: print(output_string) except (ParsingError, EvalException, MathException, TokenizationError) as e: print("ERROR: ", str(e)) def eval_string(self, string: str) -> str: """ :param string: input to interpreter :return: output as string """ tokens = self._tokenizer.tokenize(string) objs = self._parser.parse(tokens) op_type, left, right = self._recognize_operation_type(objs) if op_type == "assignment": eval_res = self._make_assignment(left, right) elif op_type == "evaluation": eval_res = Expression(left).evaluate(self._variables, self._functions) elif op_type == "equation": equation = Equation(left, right[:-1], self._variables, self._functions) eval_res = equation.solve() elif op_type == "special": spec_comm = SPECIAL_COMMANDS[left[0].name] eval_res = spec_comm.evaluate(left[0].input, self._variables, self._functions) elif op_type == "print_func": # kostyl if left[0].name not in self._functions: raise FunctionNotExists(left[0].name) f = self._functions[left[0].name] return str(f.body) else: raise Exception("Shouldn't be here man") return str(eval_res) if eval_res else None def _make_assignment(self, left: List, right: List) -> str: """ Tries to assign right part to left :param left: list of objects on left part of assignment :param right: list of objects on right part of assignment :return: string which describes assignment """ if len(left) != 1: raise WrongAssingmentLeftPart(left[0] if len(left) else None) left = left[0] if isinstance(left, Variable): right_part_evaluated = Expression(right).evaluate(self._variables, self._functions) left.val = right_part_evaluated self._variables[left.name] = left output = right_part_evaluated elif isinstance(left, AFunction): if len(left.input) != 1 or not isinstance(left.input.body[0], Variable): raise WrongAssingmentLeftPart(left.input) left_input_variable = left.input.body[0] func_body = Expression(right) if self._func_body_is_recursive(left.name, right): raise FunctionIsRecursive(left.name) func_body.evaluate_variables(self._variables, exceptions=[left_input_variable]) left.body = func_body left.input = left_input_variable self._functions[left.name] = left output = str(left) else: raise WrongAssingmentLeftPart(left) return output def _func_body_is_recursive(self, func_name, func_body): """ Checks if func_body contains any calls to func_name, including one that nested :param func_name: name of checked function :param func_body: list of objects :return: True or False """ for obj in func_body: if isinstance(obj, AFunction): if obj.name == func_name: return True if self._func_body_is_recursive(func_name, obj.input.body): return True if obj.name in self._functions and isinstance(self._functions[obj.name], UserDefinedFunction): if self._func_body_is_recursive(func_name, self._functions[obj.name].body.body): return True return False @staticmethod def _recognize_operation_type(expr: List) \ -> Tuple[str, List, List]: """ Goes through expression, tries to find strange errors and recognize, what type of expression this is: "evaluation", "assignment" or "equation" :param expr: list with operators and operands :return: (one of "evaluation", "assignment" or "equation", list of expression parts, splitted by "=" operator) """ assignment_indices = [] question_mark = False # find assignment operators and question marks. Raise exception if question mark not at the end for i, obj in enumerate(expr): if isinstance(obj, Operator) and obj.op == "=": assignment_indices.append(i) elif isinstance(obj, Operator) and obj.op == "?": if i != len(expr)-1: raise UnexpectedToken(obj.op) question_mark = True if len(assignment_indices) > 1: raise TooManyAssignments() # split input in two parts by assignment operator if not assignment_indices: left, right = expr, None else: left = expr[:assignment_indices[0]] right = expr[assignment_indices[0]+1:] if Interpreter._is_special_command(left, right): op_type = "special" elif (question_mark and len(right) == 1 and len(left) == 1 and isinstance(left[0], AFunction) and len(left[0].input.body) == 1 and isinstance(left[0].input.body[0], Variable)): op_type = "print_func" # stupid case for function definition printing elif right is None or (question_mark and len(right) == 1): op_type = "evaluation" # 'expression = ?' or no assignment operator line elif question_mark and len(right) > 1: op_type = "equation" elif assignment_indices and not question_mark: op_type = "assignment" else: raise Exception("Shouldn't be here man") return op_type, left, right @staticmethod def _is_special_command(left, right): special_funcs = [] for obj in left: if isinstance(obj, AFunction) and obj.name in SPECIAL_COMMANDS: special_funcs.append(obj) if special_funcs: if len(left) > 1 or right is not None: raise WrongSpecialCommandUse() return True else: return False
class InterpreterTest(unittest.TestCase): def setUp(self) -> None: self.source = TestSource() self.lexer = TestLexer(self.source) self.parser = Parser(self.lexer) self.interpreter = Interpreter() def interpret(self, text): self.source.put_text(text) self.lexer.lex() ast = self.parser.parse() return self.interpreter.interpret(ast) def test_interpreting_int_value(self): result = self.interpret('1;') self.assertEqual('1', str(result)) def test_interpreting_double_value(self): result = self.interpret('1.0;') self.assertEqual('1.0', str(result)) def test_interpreting_string_value(self): result = self.interpret('"hakuna matata";') self.assertEqual('hakuna matata', str(result)) def test_interpreting_bool_value(self): result = self.interpret('true;') self.assertEqual('true', str(result)) def test_interpreting_phys_value(self): result = self.interpret('3&|m/s|;') self.assertEqual('3*(m^1/s^1)', str(result)) def test_interpreting_unit_value(self): result = self.interpret('|m/s*s*s|;') self.assertEqual('(m^1/s^3)', str(result)) def test_interpreting_if_statement(self): statement = """ int x = 5; if (x==5){ bool y = true; } else{ bool y=false; } y; """ result = self.interpret(statement) self.assertEqual('true', str(result)) def test_interpreting_while_statement(self): statement = """ int counter = 1; int x = 2; while(counter < 5){ x = x * 2; counter = counter + 1; } x; """ result = self.interpret(statement) self.assertEqual('32', str(result)) def test_interpreting_function_statement(self): statement = """ function multiply_phys_values(a:phys,b:phys)->phys{ return a*b; } phys x = 1&|m/s|; phys y = 3&|m/s*s|; phys result = multiply_phys_values(x,y); result; """ result = self.interpret(statement) self.assertEqual('3*(m^2/s^3)', str(result)) def test_interpreting_multiple_ifs_statement(self): statement = """ int x = 5; int result = 0; if (x < 4){ result = 1; } elseif(x > 6){ result = 2; } elseif (x==5){ result = 3; } result; """ result = self.interpret(statement) self.assertEqual('3', str(result))
def main(): parser = argparse.ArgumentParser(description="Esoteric C compiler") parser.add_argument('infiles', metavar='infile', type=str, nargs='+', help="Input files, can be either C or ASM") parser.add_argument('-o', dest='outfile', metavar='outfile', type=str, default='a.out', required=False, help="Place the output into <outfile>") parser.add_argument( '-E', dest='preprocess_only', action='store_const', const=True, default=False, help="Preprocess only; do not compile, assemble or link.") parser.add_argument('-S', dest='compile_only', action='store_const', const=True, default=False, help="Compile only; do not assemble or link.") parser.add_argument('-c', dest='assemble_only', action='store_const', const=True, default=False, help="Compile and assemble, but do not link.") parser.add_argument('-D', dest='defines', metavar='macro[=val]', nargs=1, action='append', help='Predefine name as a macro [with value]') parser.add_argument('-I', dest='includes', metavar='path', nargs=1, action='append', help="Path to search for unfound #include's") parser.add_argument('--dump-ir', dest='dump_ir', action='store_const', const=True, default=False, help="Dump the IR into a file") parser.add_argument('--dump-ast', dest='dump_ast', action='store_const', const=True, default=False, help="Dump the AST into a file") args = parser.parse_args() ################################################ # preprocess all files ################################################ preprocessor = pcpp.Preprocessor() preprocessor.add_path('.') if args.includes is not None: for path in args.includes: preprocessor.add_path(path) # TODO: pass defines # # Figure all the files # files = [] asms = [] objects = [] for file in args.infiles: if file.endswith('.c'): preprocessor.parse(open(file), file) s = StringIO() preprocessor.write(s) code = s.getvalue() files.append((code, file)) elif file.endswith('.S'): asms.append((file, open(file).read())) elif file.endswith('.o'): obj = pickle.Unpickler(open(file, 'rb')).load() objects.append((obj, file)) else: assert False, f"Unknown file extension {file}" # # If preprocess just print the preprocessed files # if args.preprocess_only: for code, file in files: print(code) return # # Compile all c files # for code, file in files: # Parse the code into an ast parser = Parser(code, filename=file) parser.parse() if parser.got_errors: exit(1) assert not parser.got_errors # Optimize the AST opt = Optimizer(parser) opt.optimize() if args.dump_ast: with open(file[:-2] + '.ast', 'w') as f: for func in opt.parser.func_list: f.write(str(func) + '\n') # Now we need to translate it into # the ir code trans = IrTranslator(parser) trans.translate() if args.dump_ir: with open(file[:-2] + '.ir', 'w') as f: p = Printer() for proc in trans.proc_list: f.write(proc.get_name() + ":\n") for inst in proc.get_body(): f.write('\t' + p.print_instruction(inst) + '\n') # Now run it through the ir translator for # the dcpu16 code_trans = Dcpu16Translator() for proc in trans.proc_list: code_trans.translate_procedure(proc) asm = code_trans.get_asm() # Run the code through the peephole optimizer optimizer = Dcpu16PeepholeOptimizer() asm = optimizer.optimize(asm) # Add externs for any unknown label for func in parser.func_list: if func.prototype: asm += f'\n.extern {func.name}\n' # Add global vars definitions for var in parser.global_vars: if var.storage == StorageClass.EXTERN: asm += f'\n.extern {var.ident.name}\n' else: if var.storage != StorageClass.STATIC: asm += f'\n.global {var.ident.name}\n' asm += f'{var.ident.name}:\n' if var.value is None: asm += f'\t.fill {var.typ.sizeof()}, 0\n' else: asm += f'\t.dw {var.value}\n' asms.append((asm, file)) # # If we only do compilation then save the assembly files # if args.compile_only: for asm, file in asms: with open(file[:-2] + '.S', 'w') as f: f.write(asm) return # # Assemble all assembly files # for asm, file in asms: asm = Dcpu16Assembler(asm, file) asm.parse() asm.fix_labels() if asm.got_errors: exit(1) assert not asm.got_errors objects.append((asm.get_object(), file)) # # If only assemble save the object files # if args.assemble_only: for obj, file in objects: pickle.Pickler(open(file[:-2] + '.o', 'wb')).dump(obj) return # # Link everything # linker = Dcpu16Linker() for obj, file in objects: linker.append_object(obj) linker.link(BinaryType.RAW) # # Output the final binary # with open(args.outfile, 'wb') as f: for word in linker.get_words(): f.write(struct.pack('>H', word))
def evaluate(self, source_type, file_path=None): lexer = StdInLexer() if source_type == 'stdin' else FileLexer( file_path) parser = Parser(lexer) ast = parser.parse() return self.interpreter.interpret(ast)
class ParserTest(unittest.TestCase): def setUp(self) -> None: self.source = TestSource() self.lexer = TestLexer(self.source) self.parser = Parser(self.lexer) def parse(self, text): self.source.put_text(text) self.lexer.lex() return self.parser.parse() def test_parsing_int_value(self): result = self.parse('1;') self.assertEqual('(int value:1)', str(result)) def test_parsing_double_value(self): result = self.parse('1.0;') self.assertEqual('(double value:1.0)', str(result)) def test_parsing_string_value(self): result = self.parse('"text";') self.assertEqual('(string value:text)', str(result)) def test_parsing_true_value(self): result = self.parse('true;') self.assertEqual('(true)', str(result)) def test_parsing_false_value(self): result = self.parse('false;') self.assertEqual('(false)', str(result)) def test_parsing_phys_value(self): result = self.parse('3&|m/s|;') self.assertEqual('((Phys: int value:3*(Unit:m^1s^-1)))', str(result)) def test_parsing_unit_value(self): result = self.parse('|m*m/s*n*x|;') self.assertEqual('((Unit:m^2s^-1n^-1x^-1))', str(result)) def test_parsing_while_statement_value(self): result = self.parse('while (true) {1;}') self.assertEqual('((While: true Do:(int value:1)))', str(result)) def test_parsing_if_statement_value(self): result = self.parse('if (false) {"x";} else{3;}') self.assertEqual('((If:(false, (string value:x))(int value:3)))', str(result)) def test_parsing_variable_assignment_value(self): result = self.parse('int x = 5;') self.assertEqual('((Assignment: int identifier:x=int value:5))', str(result)) def test_parsing_variable_access_value(self): result = self.parse('x;') self.assertEqual('(identifier:x)', str(result)) def test_parsing_function_definition(self): result = self.parse('function add (a:int, b:int)->int{ return a+b; }') self.assertEqual( '((Function:identifier:add->int Args:[(identifier:a:int), (identifier:b:int)]' ' Body:(<Return> (identifier:a+identifier:b))))', str(result)) def test_parsing_type_int(self): result = self.parse('int x = 4;') self.assertEqual('((Assignment: int identifier:x=int value:4))', str(result)) def test_parsing_type_string(self): result = self.parse('string s = "string";') self.assertEqual( '((Assignment: string identifier:s=string value:string))', str(result)) def test_parsing_type_bool(self): result = self.parse('bool v = true;') self.assertEqual('((Assignment: bool identifier:v=true))', str(result)) def test_parsing_type_double(self): result = self.parse('double v = 2.0;') self.assertEqual( '((Assignment: double identifier:v=double value:2.0))', str(result)) def test_parsing_type_phys(self): result = self.parse('phys v = 3&|m/s|;') self.assertEqual( '((Assignment: phys identifier:v=(Phys: int value:3*(Unit:m^1s^-1))))', str(result)) def test_parsing_type_unit(self): result = self.parse('unit u = |a*s*x/c|;') self.assertEqual( '((Assignment: unit identifier:u=(Unit:a^1s^1x^1c^-1)))', str(result))