def test_fn(self):
x = "val it : int = " \
"let " \
"val x : int = 10 " \
"val double : int -> int = fn x : int => mul { 1 = x, 2 = 2 } " \
"in double x end"
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
x = "val it : int = " \
"let " \
"val x : int = 10 " \
"val sum: {1 : int, 2 : int, 3: int} -> int = " \
"fn {1 = x : int, 2 = y : int, 3 = z : int} : int => " \
"add { 1 = add { 1 = x, 2 = y} , 2 = z} " \
"in sum {1=x, 2=x, 3=x} end"
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
self.assertEqual(True, True)
def test_process_handles_correct_files_in_correct_order(self):
parser.parse(self.root)
items = sys.stdout.getvalue()
items = items.strip().split('\n')
items = [l.strip() for l in items if (l.strip() and not l.startswith(' '))]
items = ', '.join(items)
#self.old_stdout.write('"%s"\n' % items)
self.assertEquals('d1/f1.l, d1/f2.l, d1/f3.l, d2/f1.l, d2/f2.l, d2/f3.l', items)
def test_call_error(self):
x = parser.parse('val it : int = let val s : string = "Hello World!\n" in print 1; 0 end')
with self.assertRaises(SMLSyntaxError):
typecheck(x)
x = parser.parse('val it : int = let val a : int = 1.0 val s : string = "Hello World!\n" in print s; 0 end')
with self.assertRaises(SMLSyntaxError):
typecheck(x)
self.assertEqual(True, True)
def parse_intern(data, create_logger=False, expand_loops=True, expand_generators=True):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = TypeSpace()
"""
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace);
file = open(glob.g_outfile+".manifest", "w")
file.write(buf)
file.close()
"""
expand_typed_classes(result, typespace)
if glob.g_clear_slashr:
print("\n")
flatten_statementlists(result, typespace)
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
# if glob.g_enable_static_vars:
# process_static_vars(result, typespace)
buf = gen_cpp(result, typespace)
if glob.g_outfile == "":
print("\n")
print(buf)
return buf, result
def exec_do_cmd(cmd):
file_name = cmd.file_name
from lex import lexer
from parse import parser
source = open("./" + file_name).read()
ast = parser.parse(lexer.lex(source))
exec_cmd_block(ast)
def test_fun_xxx(self):
x = ''' val it = let val f : int -> int = fn 0 :int => 0 | x : int => addi { 1 = x, 2 = 10 } in print (intToStr (f 1)); 0 end '''
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
def compile(args):
switches = [arg for arg in args if arg.startswith('-')]
targets = [arg for arg in args if not arg.startswith('-')]
if not targets:
targets = ['.']
exit_code = 0
for target in targets:
exit_code += parser.parse(target)
return exit_code
def test_fun_add(self):
print("--------Code Generator Test----------")
x = 'val it : int = let val x = 10 in (print (intToStr (addi {1=x, 2=4})); 0) end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def out(source):
try:
ast2 = parser.parse(lexer.lex(source))
vistor.exec_cmd_block(ast2)
del ast2
except myerr.ParserError as e:
print >> sys.stderr, e.message
except Exception, e:
print >> sys.stderr, e
print logging.exception("The stack")
def out(source):
try:
ast2 = parser.parse(lexer.lex(source))
vistor.exec_cmd_block(ast2)
del ast2
except myerr.ParserError as e:
print >>sys.stderr , e.message
except Exception ,e:
print >>sys.stderr, e
print logging.exception("The stack")
def test_gen_hello(self):
# x = 'val it : int = let val s : string = "Hello World!\n" in print s; 0 end'
# print("--------Code Generator Test----------")
# x = 'val it : int = 0'
# print("Test: ", x)
# x = parser.parse(x)
# env = typecheck(x)
# desent(0, x)
# codeGen(x, env)
# self.assertEqual(True, True)
# print("--------Code Generator Test Finished----------")
# print("--------Code Generator Test----------")
# x = 'val it : int = let val x : int = 110 val s : string = "Hello World!\n" in x end'
# print("Test: ", x)
# x = parser.parse(x)
# env = typecheck(x)
# desent(0, x)
# codeGen(x, env)
# self.assertEqual(True, True)
# print("--------Code Generator Test Finished----------")
# print("--------Code Generator Test----------")
# x = 'val it : int = let val x : int = 110 in let val q : int = 2 in x end end'
# print("Test: ", x)
# x = parser.parse(x)
# env = typecheck(x)
# desent(0, x)
# codeGen(x, env)
# self.assertEqual(True, True)
# print("--------Code Generator Test Finished----------")
# print("--------Code Generator Test----------")
# x = 'val it : int = let val s : string = "Hello World\n" in ' \
# 'print s; let val s : string = "Goodbye!\n" in print s end; 10 end'
# print("Test: ", x)
# x = parser.parse(x)
# env = typecheck(x)
# desent(0, x)
# codeGen(x, env)
# self.assertEqual(True, True)
# print("--------Code Generator Test Finished----------")
print("--------Code Generator Test----------")
x = 'val it : int = let val {1 = x : int, 2 = y : int, ' \
'3 = {1 = a : int, 2 = b : int} : {1 : int, 2 : int} } : ' \
'{1:int, 2:int, 3: {1:int, 2:int}} = {1 = 3, 2 = 6, 3 = {1 = 10, 2 = 9} } in y end '
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_gen_std(self):
print("--------Code Generator Test----------")
x = 'val it : int = let val {x = x: real, y = y: int, z = z: string } = ' \
'{x = 3.3, y = 10, z = "abcd\n"} in print (realToStr x); print (intToStr y); print z; 0 end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_fun_rec(self):
print("--------Code Generator Test----------")
x = ''' val it =
let val rec f : int -> int =
fn 0 => 1
| x : int => muli {1 = x, 2 = f (subi {1 = x, 2 = 1})}
in print (intToStr (f 6)); 0 end '''
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_fun_bug(self):
print("--------Code Generator Test----------")
x = ''' val it =
let val f : { 1 : int, 2 : int } -> int =
fn {1 = 5, 2 = 10} => 10
| {1 = x : int, 2 = 20} => addi {1 = 20, 2 = x}
in print (intToStr (f {1 = 15, 2 = 20})); 0 end '''
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_record_assign(self):
x = 'val it : int = let val {x = a : int, y = b : real} = {x = 1, y = 2.0} in 0 end'
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
x = 'val it : int = let val x : {x : int, y : real} = {x = 1, y = 2.0} in 0 end'
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
x = 'val it : int = let ' \
'val a : int = 10 ' \
'val x : {x : int, y : real} = {x = a, y = 2.0}' \
'in x; 0 end'
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
x = 'val it : int = let ' \
'val a : int = 10 ' \
'val x : {x : int, y : {a : int, b : int}} = {x = a, y = {a = 2, b = 3}} ' \
'in x; 0 end'
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
self.assertEqual(True, True)
def test_fun_xyz(self):
print("--------Code Generator Test----------")
x = ''' val it =
let val f : { x : int, y : int, z : int } -> int =
fn {x = 5, y = 10, z = 5} => 10
| {x = x : int, y = 20, z = z : int} => addi {1 = z, 2 = x}
| {x = x : int, ...} => muli {1 = x, 2 = 2}
in print (intToStr (f {x = 20, y = 21, z = 101})); 0 end '''
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def parser_thread():
source = Sources(db.get_connection())
user_table = Users(db.get_connection())
subscription = Subscription(db.get_connection())
while True:
try:
user_list = user_table.get_all()
sources = source.get_all()
for s in sources:
id_, name, url, context = s
new_context = parse(url.strip())
if new_context:
new_context = new_context.replace("'", "")
if new_context != context and new_context:
print("new post!\n", new_context)
source.get(source_id=id_)
source.update_context(new_context)
subscription.select({'source_id': id_})
print("subscription.last_response:",
subscription.last_response)
for sub in subscription.last_response:
sub_id, source_id, user_id, data = sub
user_table.get_user(user_id=user_id)
tg_id = user_table.last_response[0][2]
url = f"https://api.telegram.org/" \
f"bot{Configuration.TOKEN}/" \
f"sendMessage?chat_id={tg_id}" \
f"&text={new_context}"
requests.get(url)
for user in user_list:
user_id, join, tg_id = user
# bot.send_message()
print(
f"NEW!! \n{new_context}\n time: {datetime.time( datetime.now())} \n------------------"
)
sleep(Configuration.parsing_interval)
except Exception as e:
print(e)
write_log(e)
def test_fun_record(self):
print("--------Code Generator Test----------")
# x = 'val it : int = let val f : int -> int = fn 0=>7 | 7=>14 | 14=>21 | x:int =>addi {1=x,2=1} in print (intToStr (f(f (f (f 0)))));0 end'
# x = 'val it : int = let val f : int -> int = fn 0=>7 | x:int => addi {1=x,2=1} in print (intToStr (f 17));0 end'
x = 'val it : int = \
let val f : {1:int ,2:int} -> int = \
fn {1=5 , 2=10} => 15 | \
{1=x:int , 2=10} => (print (intToStr x); addi{1=x,2=10}) | \
{2=10,...} =>10 | \
_ => 100 \
in print (intToStr (f {1=3,2=10}));0 end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_datatype(self):
print("--------Code Generator Test----------")
# x = ' val it = ({1=2};1;0;10)'
x = ' val it = let datatype Card = A | B of int | C of {1 : int, 2 : int} in A; 0 end'
# x = 'val it = let val a = 1 val b = 2 in 0 end'
# x = ''' val it =
# let val f : { x : int, y : int, z : int } -> int =
# fn {x = 5, y = 10, z = 5} => 10
# | {x = x : int, y = 20, z = z : int} => addi {1 = z, 2 = x}
# | {x = x : int, ...} => muli {1 = x, 2 = 2}
# in print (intToStr (f {x = 20, y = 21, z = 101})); 0 end '''
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_gen_hello(self):
print("--------Code Generator Test----------")
x = 'val it : int = \
let val f : {1:int ,2:int} -> int = \
fn {1=5 , 2=10} => 15 | \
{1=x:int , 2=10} =>addi{1=x,2=10} | \
{2=10,...} =>addi{1=5,2=20} | \
_ => 100 \
in print (intToStr (f {1=5,2=10}));0 end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
print("-------------------*******------------------------*******---------------------")
print(env)
desent(0, x)
# codeGen(x, env)
self.assertEqual(True, True)
print("--------Code Generator Test Finished----------")
def test_tyinfer(self):
x = 'val it = 0'
print("Test: ", x)
x = parser.parse(x)
typecheck(x)
desent(0, x)
x = 'val it : int = let val {x = a, y = b} = {x = 1, y = 2.0} in 0 end'
print("Test: ", x)
x = parser.parse(x)
typecheck(x)
desent(0, x)
x = 'val it = let val x = 1 val y = x in x end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
x = 'val it = let val x : {1:int, 2:real, 3:string} = {1 = 1, 2 = 2.0, 3 = "abc"} in 0 end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
x = 'val it = let val x = {1 = 1, 2 = 2.0, 3 = "abc"} in 0 end'
print("Test: ", x)
x = parser.parse(x)
env = typecheck(x)
desent(0, x)
x = 'val it = let ' \
'val a = 10 ' \
'val {x = x, y = {a = b, b = c}} = {x = a, y = {a = 2, b = 3}} ' \
'in c end'
print("Test: ", x)
x = parser.parse(x)
# desent(0, x)
typecheck(x)
desent(0, x)
self.assertEqual(True, True)
def dt_parse(
data,
args=None,
file="",
flatten=True,
print_stack=True,
start_node=None,
print_warnings=False,
exit_on_err=True,
log_productions=False,
validate=False,
):
back = glob.copy()
def safe_get(data, i):
if i < 0:
return ""
elif i >= len(data):
return ""
return data[i]
if args != None:
if not isinstance(args, tuple) and not isinstance(args, list):
if caniter(args) and not isinstance(args, Node) and type(args) not in [str, bytes]:
args = list(args)
else:
args = (args,) # encapsulate single arguments in a tuple
i = 0
ai = 0
while i < len(data) - 2:
if data[i] == "$" and safe_get(data, i - 1) != "$":
i1 = i
t = data[i + 1]
i += 2
arg, i = fetch_int(data, i)
if arg == None:
arg = ai
ai += 1
else:
arg -= 1
ai = max(arg, ai)
if arg >= len(args):
raise RuntimeError("Not enough args for format conversion in parse()")
if t == "n":
buf = args[arg].gen_js(0)
elif t == "s":
buf = str(args[arg])
elif t in ["d", "i", "u"]:
buf = str(int(args[arg]))
elif t in ["f", "lf"]:
buf = str(float(args[arg]))
elif t == "x":
buf = hex(int(args[arg]))
else:
buf = data[i1:i]
data = data[:i1] + buf + data[i:]
i = i1
i += 1
glob.reset()
glob.g_exit_on_err = exit_on_err
glob.g_lexer = plexer
glob.g_production_debug = False
glob.g_file = file
glob.g_print_stack = print_stack
glob.g_print_warnings = print_warnings
glob.g_log_productions = log_productions
glob.g_validate_mode = validate
plexer.lineno = plexer.lexer.lineno = 0
plexer.input(data)
ret = parser.parse(data, lexer=plexer)
if glob.g_error:
print("------------LLLLLLLLLLLLLLLLLLL yeek!!!")
ret = None
if glob.g_clear_slashr:
print("\n")
def fix_parents(node, lastnode=None):
if node.parent in [0, None]:
node.parent = lastnode
for c in node.children:
fix_parents(c, node)
if ret != None:
fix_parents(ret)
if flatten:
ret = flatten_statementlists(ret, None)
if ret == None:
traceback.print_stack()
sys.stderr.write("error: internal parse error within parse\n")
sys.exit(-1)
if start_node != None and ret != None:
def visit(n):
if type(n) == start_node:
return n
for c in n.children:
c2 = visit(c)
if c2 != None:
return c2
ret = visit(ret)
if ret != None:
combine_try_nodes(ret)
glob.load(back)
return ret
def runsource(self, source, filename="<input>", symbol="single"):
ast = parser.parse(lexer.lex(source))
vistor.exec_cmd_block(ast)
def compile():
input_text = read_file("input.txt")
parse(input_text)
def parse_intern(data,
create_logger=False,
expand_loops=True,
expand_generators=True):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = TypeSpace()
"""
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace);
file = open(glob.g_outfile+".manifest", "w")
file.write(buf)
file.close()
"""
expand_typed_classes(result, typespace)
if glob.g_clear_slashr:
print("\n")
flatten_statementlists(result, typespace)
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
#if glob.g_enable_static_vars:
# process_static_vars(result, typespace)
buf = gen_cpp(result, typespace)
if glob.g_outfile == "":
print("\n")
print(buf)
return buf, result
def dt_parse(data,
args=None,
file="",
flatten=True,
print_stack=True,
start_node=None,
print_warnings=False,
exit_on_err=True,
log_productions=False,
validate=False):
back = glob.copy()
def safe_get(data, i):
if i < 0: return ""
elif i >= len(data): return ""
return data[i]
if args != None:
if not isinstance(args, tuple) and not isinstance(args, list):
if caniter(args) and not isinstance(args, Node) \
and type(args) not in [str, bytes]:
args = list(args)
else:
args = (args, ) #encapsulate single arguments in a tuple
i = 0
ai = 0
while i < len(data) - 2:
if data[i] == "$" and safe_get(data, i - 1) != "$":
i1 = i
t = data[i + 1]
i += 2
arg, i = fetch_int(data, i)
if arg == None:
arg = ai
ai += 1
else:
arg -= 1
ai = max(arg, ai)
if arg >= len(args):
raise RuntimeError(
"Not enough args for format conversion in parse()")
if t == "n":
buf = args[arg].gen_js(0)
elif t == "s":
buf = str(args[arg])
elif t in ["d", "i", "u"]:
buf = str(int(args[arg]))
elif t in ["f", "lf"]:
buf = str(float(args[arg]))
elif t == "x":
buf = hex(int(args[arg]))
else:
buf = data[i1:i]
data = data[:i1] + buf + data[i:]
i = i1
i += 1
glob.reset()
glob.g_exit_on_err = exit_on_err
glob.g_lexer = plexer
glob.g_production_debug = False
glob.g_file = file
glob.g_print_stack = print_stack
glob.g_print_warnings = print_warnings
glob.g_log_productions = log_productions
glob.g_validate_mode = validate
plexer.lineno = plexer.lexer.lineno = 0
plexer.input(data)
ret = parser.parse(data, lexer=plexer)
if glob.g_error:
print("------------LLLLLLLLLLLLLLLLLLL yeek!!!")
ret = None
if glob.g_clear_slashr:
print("\n")
def fix_parents(node, lastnode=None):
if node.parent in [0, None]:
node.parent = lastnode
for c in node.children:
fix_parents(c, node)
if ret != None:
fix_parents(ret)
if flatten:
ret = flatten_statementlists(ret, None)
if ret == None:
traceback.print_stack()
sys.stderr.write("error: internal parse error within parse\n")
sys.exit(-1)
if start_node != None and ret != None:
def visit(n):
if type(n) == start_node:
return n
for c in n.children:
c2 = visit(c)
if c2 != None:
return c2
ret = visit(ret)
if ret != None:
combine_try_nodes(ret)
glob.load(back)
return ret
def _compile(s):
r = Ast(parser.parse(s))
c = codegen(r, [])
code = handle_callcc(c+['STOP'])
return code
_eval(_compile(s_import))
_eval(_compile('import("lib.py")'))
#_eval(_compile(s_repl))
#_eval(_compile('repl()'))
#from prompt_toolkit import prompt
import pdb
if __name__ == '__main__':
args = sys.argv
if not ('-d' in args):
# normal mode
G[__debug__] = False
while True:
try:
s=input(colored('pure> ', "cyan"))
if not s:continue
result = Ast(parser.parse(s))
code = handle_callcc(codegen(result, []) + ['STOP'])
#c = optimize_global(code)
V=_eval(code)
if V[0] is None:continue
print(V[0])
except Exception as e:
cprint(e.__class__.__name__ + ':' + str(e), "red")
#print(e)
continue
# debug mode
G[__debug__] = True
pdb.set_trace()
while True:
s = input(colored('debug> ', 'red'))
if not s:continue
def compiles(string):
generated_ast = python_ast.generate(tree.ast(parser.parse(lexer.lexs(string))))
return compile(generated_ast, '<test>', 'exec')
def test_hello(self):
x = parser.parse('val it : int = let val s : string = "Hello World!\n" in print s; 0 end')
# desent(0, x)
typecheck(x)
desent(0, x)
self.assertEqual(True, True)
def test_ast(self):
x = parser.parse(
"val it : int = let val x : int = 10 val double : int -> int = fn x : int => x mul 2 in double x end")
desent(0, x)
self.assertEqual(True, True)
#!/usr/local/bin/python3
__author__ = 'Xing, Chang'
import sys
from parse import parser
from parse import errflag
from typecheck import *
from codegen import *
if __name__ == "__main__":
f = open(sys.argv[1])
src = f.read()
x = parser.parse(src)
env = typecheck(x)
desent(0, x)
if not errflag[0]:
codeGen(x, env, sys.argv[1].split(".")[0] + ".ll")
def parsing(source_code):
parser.parse(source_code)
def test_sample_sandbox(self): my_folder = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(my_folder, '..', '..', 'samples', 'traffic' + sandbox.SUFFIX) parser.parse(path)
