def test_fn(self): x = "val it : int = " \ "let " \ "val x : int = 10 " \ "val double : int -> int = fn x : int => mul { 1 = x, 2 = 2 } " \ "in double x end" print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) x = "val it : int = " \ "let " \ "val x : int = 10 " \ "val sum: {1 : int, 2 : int, 3: int} -> int = " \ "fn {1 = x : int, 2 = y : int, 3 = z : int} : int => " \ "add { 1 = add { 1 = x, 2 = y} , 2 = z} " \ "in sum {1=x, 2=x, 3=x} end" print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) self.assertEqual(True, True)
def test_process_handles_correct_files_in_correct_order(self): parser.parse(self.root) items = sys.stdout.getvalue() items = items.strip().split('\n') items = [l.strip() for l in items if (l.strip() and not l.startswith(' '))] items = ', '.join(items) #self.old_stdout.write('"%s"\n' % items) self.assertEquals('d1/f1.l, d1/f2.l, d1/f3.l, d2/f1.l, d2/f2.l, d2/f3.l', items)
def test_call_error(self): x = parser.parse('val it : int = let val s : string = "Hello World!\n" in print 1; 0 end') with self.assertRaises(SMLSyntaxError): typecheck(x) x = parser.parse('val it : int = let val a : int = 1.0 val s : string = "Hello World!\n" in print s; 0 end') with self.assertRaises(SMLSyntaxError): typecheck(x) self.assertEqual(True, True)
def parse_intern(data, create_logger=False, expand_loops=True, expand_generators=True): glob.g_lines = data.split("\n") if glob.g_preprocess_code: data = preprocess_text(data, glob.g_file) if glob.g_print_tokens: plexer.input(data) tok = plexer.token() while tok != None: print(tok) tok = plexer.token() plexer.input(data) glob.g_lexer = plexer result = parser.parse(data, lexer=plexer) if result == None: if glob.g_error_pre != None: glob.g_error = True result = StatementList() if glob.g_error: print_err(glob.g_error_pre) typespace = TypeSpace() """ if glob.g_write_manifest and glob.g_outfile != "": buf = gen_manifest_file(result, typespace); file = open(glob.g_outfile+".manifest", "w") file.write(buf) file.close() """ expand_typed_classes(result, typespace) if glob.g_clear_slashr: print("\n") flatten_statementlists(result, typespace) if glob.g_combine_ifelse_nodes: combine_if_else_nodes(result) if glob.g_print_nodes: print("nodes: ", result) pass # if glob.g_enable_static_vars: # process_static_vars(result, typespace) buf = gen_cpp(result, typespace) if glob.g_outfile == "": print("\n") print(buf) return buf, result
def exec_do_cmd(cmd): file_name = cmd.file_name from lex import lexer from parse import parser source = open("./" + file_name).read() ast = parser.parse(lexer.lex(source)) exec_cmd_block(ast)
def test_fun_xxx(self): x = ''' val it = let val f : int -> int = fn 0 :int => 0 | x : int => addi { 1 = x, 2 = 10 } in print (intToStr (f 1)); 0 end ''' x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True)
def compile(args): switches = [arg for arg in args if arg.startswith('-')] targets = [arg for arg in args if not arg.startswith('-')] if not targets: targets = ['.'] exit_code = 0 for target in targets: exit_code += parser.parse(target) return exit_code
def test_fun_add(self): print("--------Code Generator Test----------") x = 'val it : int = let val x = 10 in (print (intToStr (addi {1=x, 2=4})); 0) end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def out(source): try: ast2 = parser.parse(lexer.lex(source)) vistor.exec_cmd_block(ast2) del ast2 except myerr.ParserError as e: print >> sys.stderr, e.message except Exception, e: print >> sys.stderr, e print logging.exception("The stack")
def out(source): try: ast2 = parser.parse(lexer.lex(source)) vistor.exec_cmd_block(ast2) del ast2 except myerr.ParserError as e: print >>sys.stderr , e.message except Exception ,e: print >>sys.stderr, e print logging.exception("The stack")
def test_gen_hello(self): # x = 'val it : int = let val s : string = "Hello World!\n" in print s; 0 end' # print("--------Code Generator Test----------") # x = 'val it : int = 0' # print("Test: ", x) # x = parser.parse(x) # env = typecheck(x) # desent(0, x) # codeGen(x, env) # self.assertEqual(True, True) # print("--------Code Generator Test Finished----------") # print("--------Code Generator Test----------") # x = 'val it : int = let val x : int = 110 val s : string = "Hello World!\n" in x end' # print("Test: ", x) # x = parser.parse(x) # env = typecheck(x) # desent(0, x) # codeGen(x, env) # self.assertEqual(True, True) # print("--------Code Generator Test Finished----------") # print("--------Code Generator Test----------") # x = 'val it : int = let val x : int = 110 in let val q : int = 2 in x end end' # print("Test: ", x) # x = parser.parse(x) # env = typecheck(x) # desent(0, x) # codeGen(x, env) # self.assertEqual(True, True) # print("--------Code Generator Test Finished----------") # print("--------Code Generator Test----------") # x = 'val it : int = let val s : string = "Hello World\n" in ' \ # 'print s; let val s : string = "Goodbye!\n" in print s end; 10 end' # print("Test: ", x) # x = parser.parse(x) # env = typecheck(x) # desent(0, x) # codeGen(x, env) # self.assertEqual(True, True) # print("--------Code Generator Test Finished----------") print("--------Code Generator Test----------") x = 'val it : int = let val {1 = x : int, 2 = y : int, ' \ '3 = {1 = a : int, 2 = b : int} : {1 : int, 2 : int} } : ' \ '{1:int, 2:int, 3: {1:int, 2:int}} = {1 = 3, 2 = 6, 3 = {1 = 10, 2 = 9} } in y end ' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_gen_std(self): print("--------Code Generator Test----------") x = 'val it : int = let val {x = x: real, y = y: int, z = z: string } = ' \ '{x = 3.3, y = 10, z = "abcd\n"} in print (realToStr x); print (intToStr y); print z; 0 end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_fun_rec(self): print("--------Code Generator Test----------") x = ''' val it = let val rec f : int -> int = fn 0 => 1 | x : int => muli {1 = x, 2 = f (subi {1 = x, 2 = 1})} in print (intToStr (f 6)); 0 end ''' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_fun_bug(self): print("--------Code Generator Test----------") x = ''' val it = let val f : { 1 : int, 2 : int } -> int = fn {1 = 5, 2 = 10} => 10 | {1 = x : int, 2 = 20} => addi {1 = 20, 2 = x} in print (intToStr (f {1 = 15, 2 = 20})); 0 end ''' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_record_assign(self): x = 'val it : int = let val {x = a : int, y = b : real} = {x = 1, y = 2.0} in 0 end' print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) x = 'val it : int = let val x : {x : int, y : real} = {x = 1, y = 2.0} in 0 end' print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) x = 'val it : int = let ' \ 'val a : int = 10 ' \ 'val x : {x : int, y : real} = {x = a, y = 2.0}' \ 'in x; 0 end' print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) x = 'val it : int = let ' \ 'val a : int = 10 ' \ 'val x : {x : int, y : {a : int, b : int}} = {x = a, y = {a = 2, b = 3}} ' \ 'in x; 0 end' print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) self.assertEqual(True, True)
def test_fun_xyz(self): print("--------Code Generator Test----------") x = ''' val it = let val f : { x : int, y : int, z : int } -> int = fn {x = 5, y = 10, z = 5} => 10 | {x = x : int, y = 20, z = z : int} => addi {1 = z, 2 = x} | {x = x : int, ...} => muli {1 = x, 2 = 2} in print (intToStr (f {x = 20, y = 21, z = 101})); 0 end ''' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def parser_thread(): source = Sources(db.get_connection()) user_table = Users(db.get_connection()) subscription = Subscription(db.get_connection()) while True: try: user_list = user_table.get_all() sources = source.get_all() for s in sources: id_, name, url, context = s new_context = parse(url.strip()) if new_context: new_context = new_context.replace("'", "") if new_context != context and new_context: print("new post!\n", new_context) source.get(source_id=id_) source.update_context(new_context) subscription.select({'source_id': id_}) print("subscription.last_response:", subscription.last_response) for sub in subscription.last_response: sub_id, source_id, user_id, data = sub user_table.get_user(user_id=user_id) tg_id = user_table.last_response[0][2] url = f"https://api.telegram.org/" \ f"bot{Configuration.TOKEN}/" \ f"sendMessage?chat_id={tg_id}" \ f"&text={new_context}" requests.get(url) for user in user_list: user_id, join, tg_id = user # bot.send_message() print( f"NEW!! \n{new_context}\n time: {datetime.time( datetime.now())} \n------------------" ) sleep(Configuration.parsing_interval) except Exception as e: print(e) write_log(e)
def test_fun_record(self): print("--------Code Generator Test----------") # x = 'val it : int = let val f : int -> int = fn 0=>7 | 7=>14 | 14=>21 | x:int =>addi {1=x,2=1} in print (intToStr (f(f (f (f 0)))));0 end' # x = 'val it : int = let val f : int -> int = fn 0=>7 | x:int => addi {1=x,2=1} in print (intToStr (f 17));0 end' x = 'val it : int = \ let val f : {1:int ,2:int} -> int = \ fn {1=5 , 2=10} => 15 | \ {1=x:int , 2=10} => (print (intToStr x); addi{1=x,2=10}) | \ {2=10,...} =>10 | \ _ => 100 \ in print (intToStr (f {1=3,2=10}));0 end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_datatype(self): print("--------Code Generator Test----------") # x = ' val it = ({1=2};1;0;10)' x = ' val it = let datatype Card = A | B of int | C of {1 : int, 2 : int} in A; 0 end' # x = 'val it = let val a = 1 val b = 2 in 0 end' # x = ''' val it = # let val f : { x : int, y : int, z : int } -> int = # fn {x = 5, y = 10, z = 5} => 10 # | {x = x : int, y = 20, z = z : int} => addi {1 = z, 2 = x} # | {x = x : int, ...} => muli {1 = x, 2 = 2} # in print (intToStr (f {x = 20, y = 21, z = 101})); 0 end ''' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_gen_hello(self): print("--------Code Generator Test----------") x = 'val it : int = \ let val f : {1:int ,2:int} -> int = \ fn {1=5 , 2=10} => 15 | \ {1=x:int , 2=10} =>addi{1=x,2=10} | \ {2=10,...} =>addi{1=5,2=20} | \ _ => 100 \ in print (intToStr (f {1=5,2=10}));0 end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) print("-------------------*******------------------------*******---------------------") print(env) desent(0, x) # codeGen(x, env) self.assertEqual(True, True) print("--------Code Generator Test Finished----------")
def test_tyinfer(self): x = 'val it = 0' print("Test: ", x) x = parser.parse(x) typecheck(x) desent(0, x) x = 'val it : int = let val {x = a, y = b} = {x = 1, y = 2.0} in 0 end' print("Test: ", x) x = parser.parse(x) typecheck(x) desent(0, x) x = 'val it = let val x = 1 val y = x in x end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) x = 'val it = let val x : {1:int, 2:real, 3:string} = {1 = 1, 2 = 2.0, 3 = "abc"} in 0 end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) x = 'val it = let val x = {1 = 1, 2 = 2.0, 3 = "abc"} in 0 end' print("Test: ", x) x = parser.parse(x) env = typecheck(x) desent(0, x) x = 'val it = let ' \ 'val a = 10 ' \ 'val {x = x, y = {a = b, b = c}} = {x = a, y = {a = 2, b = 3}} ' \ 'in c end' print("Test: ", x) x = parser.parse(x) # desent(0, x) typecheck(x) desent(0, x) self.assertEqual(True, True)
def dt_parse( data, args=None, file="", flatten=True, print_stack=True, start_node=None, print_warnings=False, exit_on_err=True, log_productions=False, validate=False, ): back = glob.copy() def safe_get(data, i): if i < 0: return "" elif i >= len(data): return "" return data[i] if args != None: if not isinstance(args, tuple) and not isinstance(args, list): if caniter(args) and not isinstance(args, Node) and type(args) not in [str, bytes]: args = list(args) else: args = (args,) # encapsulate single arguments in a tuple i = 0 ai = 0 while i < len(data) - 2: if data[i] == "$" and safe_get(data, i - 1) != "$": i1 = i t = data[i + 1] i += 2 arg, i = fetch_int(data, i) if arg == None: arg = ai ai += 1 else: arg -= 1 ai = max(arg, ai) if arg >= len(args): raise RuntimeError("Not enough args for format conversion in parse()") if t == "n": buf = args[arg].gen_js(0) elif t == "s": buf = str(args[arg]) elif t in ["d", "i", "u"]: buf = str(int(args[arg])) elif t in ["f", "lf"]: buf = str(float(args[arg])) elif t == "x": buf = hex(int(args[arg])) else: buf = data[i1:i] data = data[:i1] + buf + data[i:] i = i1 i += 1 glob.reset() glob.g_exit_on_err = exit_on_err glob.g_lexer = plexer glob.g_production_debug = False glob.g_file = file glob.g_print_stack = print_stack glob.g_print_warnings = print_warnings glob.g_log_productions = log_productions glob.g_validate_mode = validate plexer.lineno = plexer.lexer.lineno = 0 plexer.input(data) ret = parser.parse(data, lexer=plexer) if glob.g_error: print("------------LLLLLLLLLLLLLLLLLLL yeek!!!") ret = None if glob.g_clear_slashr: print("\n") def fix_parents(node, lastnode=None): if node.parent in [0, None]: node.parent = lastnode for c in node.children: fix_parents(c, node) if ret != None: fix_parents(ret) if flatten: ret = flatten_statementlists(ret, None) if ret == None: traceback.print_stack() sys.stderr.write("error: internal parse error within parse\n") sys.exit(-1) if start_node != None and ret != None: def visit(n): if type(n) == start_node: return n for c in n.children: c2 = visit(c) if c2 != None: return c2 ret = visit(ret) if ret != None: combine_try_nodes(ret) glob.load(back) return ret
def runsource(self, source, filename="<input>", symbol="single"): ast = parser.parse(lexer.lex(source)) vistor.exec_cmd_block(ast)
def compile(): input_text = read_file("input.txt") parse(input_text)
def parse_intern(data, create_logger=False, expand_loops=True, expand_generators=True): glob.g_lines = data.split("\n") if glob.g_preprocess_code: data = preprocess_text(data, glob.g_file) if glob.g_print_tokens: plexer.input(data) tok = plexer.token() while tok != None: print(tok) tok = plexer.token() plexer.input(data) glob.g_lexer = plexer result = parser.parse(data, lexer=plexer) if result == None: if glob.g_error_pre != None: glob.g_error = True result = StatementList() if glob.g_error: print_err(glob.g_error_pre) typespace = TypeSpace() """ if glob.g_write_manifest and glob.g_outfile != "": buf = gen_manifest_file(result, typespace); file = open(glob.g_outfile+".manifest", "w") file.write(buf) file.close() """ expand_typed_classes(result, typespace) if glob.g_clear_slashr: print("\n") flatten_statementlists(result, typespace) if glob.g_combine_ifelse_nodes: combine_if_else_nodes(result) if glob.g_print_nodes: print("nodes: ", result) pass #if glob.g_enable_static_vars: # process_static_vars(result, typespace) buf = gen_cpp(result, typespace) if glob.g_outfile == "": print("\n") print(buf) return buf, result
def dt_parse(data, args=None, file="", flatten=True, print_stack=True, start_node=None, print_warnings=False, exit_on_err=True, log_productions=False, validate=False): back = glob.copy() def safe_get(data, i): if i < 0: return "" elif i >= len(data): return "" return data[i] if args != None: if not isinstance(args, tuple) and not isinstance(args, list): if caniter(args) and not isinstance(args, Node) \ and type(args) not in [str, bytes]: args = list(args) else: args = (args, ) #encapsulate single arguments in a tuple i = 0 ai = 0 while i < len(data) - 2: if data[i] == "$" and safe_get(data, i - 1) != "$": i1 = i t = data[i + 1] i += 2 arg, i = fetch_int(data, i) if arg == None: arg = ai ai += 1 else: arg -= 1 ai = max(arg, ai) if arg >= len(args): raise RuntimeError( "Not enough args for format conversion in parse()") if t == "n": buf = args[arg].gen_js(0) elif t == "s": buf = str(args[arg]) elif t in ["d", "i", "u"]: buf = str(int(args[arg])) elif t in ["f", "lf"]: buf = str(float(args[arg])) elif t == "x": buf = hex(int(args[arg])) else: buf = data[i1:i] data = data[:i1] + buf + data[i:] i = i1 i += 1 glob.reset() glob.g_exit_on_err = exit_on_err glob.g_lexer = plexer glob.g_production_debug = False glob.g_file = file glob.g_print_stack = print_stack glob.g_print_warnings = print_warnings glob.g_log_productions = log_productions glob.g_validate_mode = validate plexer.lineno = plexer.lexer.lineno = 0 plexer.input(data) ret = parser.parse(data, lexer=plexer) if glob.g_error: print("------------LLLLLLLLLLLLLLLLLLL yeek!!!") ret = None if glob.g_clear_slashr: print("\n") def fix_parents(node, lastnode=None): if node.parent in [0, None]: node.parent = lastnode for c in node.children: fix_parents(c, node) if ret != None: fix_parents(ret) if flatten: ret = flatten_statementlists(ret, None) if ret == None: traceback.print_stack() sys.stderr.write("error: internal parse error within parse\n") sys.exit(-1) if start_node != None and ret != None: def visit(n): if type(n) == start_node: return n for c in n.children: c2 = visit(c) if c2 != None: return c2 ret = visit(ret) if ret != None: combine_try_nodes(ret) glob.load(back) return ret
def _compile(s): r = Ast(parser.parse(s)) c = codegen(r, []) code = handle_callcc(c+['STOP']) return code
_eval(_compile(s_import)) _eval(_compile('import("lib.py")')) #_eval(_compile(s_repl)) #_eval(_compile('repl()')) #from prompt_toolkit import prompt import pdb if __name__ == '__main__': args = sys.argv if not ('-d' in args): # normal mode G[__debug__] = False while True: try: s=input(colored('pure> ', "cyan")) if not s:continue result = Ast(parser.parse(s)) code = handle_callcc(codegen(result, []) + ['STOP']) #c = optimize_global(code) V=_eval(code) if V[0] is None:continue print(V[0]) except Exception as e: cprint(e.__class__.__name__ + ':' + str(e), "red") #print(e) continue # debug mode G[__debug__] = True pdb.set_trace() while True: s = input(colored('debug> ', 'red')) if not s:continue
def compiles(string): generated_ast = python_ast.generate(tree.ast(parser.parse(lexer.lexs(string)))) return compile(generated_ast, '<test>', 'exec')
def test_hello(self): x = parser.parse('val it : int = let val s : string = "Hello World!\n" in print s; 0 end') # desent(0, x) typecheck(x) desent(0, x) self.assertEqual(True, True)
def test_ast(self): x = parser.parse( "val it : int = let val x : int = 10 val double : int -> int = fn x : int => x mul 2 in double x end") desent(0, x) self.assertEqual(True, True)
#!/usr/local/bin/python3 __author__ = 'Xing, Chang' import sys from parse import parser from parse import errflag from typecheck import * from codegen import * if __name__ == "__main__": f = open(sys.argv[1]) src = f.read() x = parser.parse(src) env = typecheck(x) desent(0, x) if not errflag[0]: codeGen(x, env, sys.argv[1].split(".")[0] + ".ll")
def parsing(source_code): parser.parse(source_code)
def test_sample_sandbox(self): my_folder = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(my_folder, '..', '..', 'samples', 'traffic' + sandbox.SUFFIX) parser.parse(path)