def __test__eat_mkw_try_specified():
assert(
__eat_mkw_try_specified(lexer.tokenize('LEFT JOIN'), ['LEFT', 'JOIN'])
== 'LEFT JOIN'
)
assert(
__eat_mkw_try_specified(lexer.tokenize('LEFT JOIN'), ['NO', 'MATCH'])
== None
)
def test_lexer(self):
e1 = "1"
self.assertEqual(list(simplify(tokenize(e1))),
[('NUMBER', 1), 'EOF'])
input_str = "(1 + 11 * 22)"
result = list(simplify(tokenize(input_str)))
expected = ['LPAREN', ('NUMBER', 1), 'PLUS', ('NUMBER', 11),
'TIMES', ('NUMBER', 22), 'RPAREN', 'EOF']
self.assertEqual(expected, result)
def test_tokenize(self):
"""
It tests the calclex.tokenize function in the calclex module.
"""
testcases = {
".32.43 1.23 3.":
('LexToken(NUMBER,0.32,1,0), LexToken(NUMBER,0.43,1,3)'
', LexToken(NUMBER,1.23,1,7), LexToken(NUMBER,3.0,1,12)'),
"x**=abc+sum(max(a,b))":
("LexToken(ID,'x',1,0), LexToken(POWER_ASSIGN,'**=',1,1)"
", LexToken(ID,'abc',1,4), LexToken(PLUS,'+',1,7)"
", LexToken(ID,'sum',1,8), LexToken(LPAREN,'(',1,11)"
", LexToken(ID,'max',1,12), LexToken(LPAREN,'(',1,15)"
", LexToken(ID,'a',1,16), LexToken(COMMA,',',1,17)"
", LexToken(ID,'b',1,18), LexToken(RPAREN,')',1,19)"
", LexToken(RPAREN,')',1,20)")}
# each token has (type, value, lineno, lexpos)
for key, val in testcases.items():
x = []
for tok in lex.tokenize(key):
x.append(str(tok))
output = ", ".join(x)
self.assertTrue(output == val)
def test_operators():
cases = [
('1 + 1', [BinOp(Num(1), '+', Num(1))]),
]
for string, ast in cases:
tokens = lexer.tokenize(string)
assert bcparser.parse(tokens) == ast
def solve(self, expr: str = ""):
"""solve cryptarithm problem"""
print("Problem: {}".format(expr))
p = Parser(tokenize(expr))
pr = Problem(p.parse())
print(pr.search_all_solution())
def __init__(self, inptline, maxhistory=2):
self.tokgen = lex.tokenize(inptline)
self.maxhistory = maxhistory
self.tokqueue = deque(maxlen=self.maxhistory)
self.tokindex = 0
self.tokstreamended = False
self.currtok = None # the current token returned
# read all the tokens untill the queue is full
# or the tokens are finished, which ever is first.
count = 0
while count < self.maxhistory:
if self.tokstreamended:
self.tokqueue.append(None)
count += 1
continue
try:
tok = next(self.tokgen)
self.tokqueue.append(tok)
except:
# signifies that the self.tokgen has run out of tokens
# None signifies that the token stream ends.
self.tokqueue.append(None)
self.tokstreamended = True
count += 1
logger.info(self.tokqueue)
self.tokstreamended = False # for nextok() to use it fresh
def main():
ast.g_llvm_pass_manager.add(ast.g_llvm_executor.target_data)
ast.g_llvm_pass_manager.add(PASS_INSTRUCTION_COMBINING)
ast.g_llvm_pass_manager.add(PASS_REASSOCIATE)
ast.g_llvm_pass_manager.add(PASS_GVN)
ast.g_llvm_pass_manager.add(PASS_CFG_SIMPLIFICATION)
ast.g_llvm_pass_manager.initialize()
operator_precedence = {
'<': 10,
'+': 20,
'-': 20,
'/': 40,
'*': 40
}
while True:
print('ready>')
try:
raw = raw_input()
except KeyboardInterrupt:
break
parser = parse.Parser(lexer.tokenize(raw), operator_precedence)
while True:
if isinstance(parser.current, lexer.EOFToken):
break
if isinstance(parser.current, lexer.DefToken):
parser.handle_definition()
elif isinstance(parser.current, lexer.ExternToken):
parser.handle_extern()
else:
parser.handle_top_level_expression()
print('\n', ast.g_llvm_module)
def compile(source, filename): # @ReservedAssignment
tokens = lexer.tokenize(source)
parser.init(tokens)
tree = parser.program()
def error(message, token, level='Error'):
if token is None:
# should be avoided for good error messages
sys.stderr.write('%s %s\n' % (level, message))
else:
lines = source.splitlines()
sys.stderr.write('%s\n%s^\n' % (lines[token.line_no-1], ' '*(token.col_no-1)))
sys.stderr.write('%s:%d:%d: %s %s\n' % (filename, token.line_no, token.col_no, level, message))
scopes.error = error
print 'S-expr:', ast.stree(tree)
print
ast.ptree(tree)
print
code, globls = scopes.build(tree)
for name, symbol in globls.names.items():
print name, symbol
print
for line in code:
print '\t'.join(map(str, line))
return code, globls
def parse_source(source):
from lexer import tokenize
toks = tokenize(source)
p = Parser(toks)
return p.parse_program()
def test_double_equals(self):
"""Test tokenizing double equals."""
self.assertEqual(lexer.tokenize("a == 10", ""), [
Token(token_kinds.identifier, "a"),
Token(token_kinds.twoequals),
Token(token_kinds.number, "10")
])
def test_symbol_splits_keywords(self):
"""Test that the lexer splits on symbols."""
self.assertEqual(lexer.tokenize("ident1+ident2", ""), [
Token(token_kinds.identifier, "ident1"),
Token(token_kinds.plus),
Token(token_kinds.identifier, "ident2")
])
def test_lexer_comment(self):
input_str = "// this is a comment\n+"
result = list(simplify(tokenize(input_str)))
expected = ['PLUS', 'EOF']
self.assertEqual(expected, result)
def parse(self, s):
self.lexer = lexer.tokenize(s)
self.Next()
r = self.ParseUntil(0)
if not self.AtToken('eof'):
raise ParseError('There are unparsed tokens: %r' % self.token)
return r
def compile_bytecode(name):
with open(name, 'r') as f:
text = f.read()
tokens = tokenize(text)
ast = parse(tokens)
fns, stack = codegen(ast)
return fns, stack
def parse_type(line_idx, line, type_str):
try:
type_tokens = lexer.tokenize(type_str)
except frontend.SileLexError, e:
raise DeserializeException(
"line {} invalid type name: {}: {}".format(
line_idx, line, e))
def test_lexer():
cases = [
("a", [Ident("a")]),
("b", [Ident("b")]),
("1", [Num(1)]),
]
for inp, outp in cases:
assert list(lexer.tokenize(inp)) == outp
def parse():
print('POEHALI')
lex = lexer.tokenize()
parser = yacc.yacc()
ast = parser.parse(lexer=lex, debug=1)
file = open('out.txt', 'w')
file.write(json.dumps(ast, indent=4))
print(json.dumps(ast, indent=4))
def test_lexer_return(self):
input_str = "return x;"
result = list(simplify(tokenize(input_str)))
expected = ['RETURN', ('ID', 'x'), 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_print(self):
input_str = "print(x);"
result = list(simplify(tokenize(input_str)))
expected = ['PRINT', 'LPAREN', ('ID', 'x'), 'RPAREN', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_continue(self):
input_str = "continue;"
result = list(simplify(tokenize(input_str)))
expected = ['CONTINUE', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_address(self):
inputs = [
("d = &n", [('ID', 'd'), 'EQUAL', 'AMPERSAND', ('ID', 'n'),
'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_break(self):
input_str = "break;"
result = list(simplify(tokenize(input_str)))
expected = ['BREAK', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_binops(self):
input_str = "x && y || z"
result = list(simplify(tokenize(input_str)))
expected = [('ID', 'x'), 'AND', ('ID', 'y'), 'OR', ('ID', 'z'), 'EOF']
self.assertEqual(expected, result)
def test_lexer_decl_pointers(self):
inputs = [
("int* d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']),
("int *d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']),
("long* d", [('TYPE', 'long'), 'TIMES', ('ID', 'd'), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_with_let(self):
input_str = "let x := 1 in x + x end"
result = list(simplify(tokenize(input_str)))
expected = ['LET', ('ID', "x"), "ASSIGN", ('NUMBER', 1), "IN",
('ID', "x"), 'PLUS', ('ID', "x"), 'END', 'EOF']
self.assertEqual(expected, result)
def main():
# Input verification.
if len(argv) < 2:
print("Error! Filename not specified!")
exit(1)
else:
if argv[1] == '-help':
print_instructions()
exit(0)
filename = argv[1]
parse_flags(argv[2:])
try:
with open(filename, 'r') as file:
lines = file.read().splitlines()
tokens = tokenize(lines)
try:
address = 0
if not flag_values['s']:
print(".text") # section
for token in tokens:
# Offset used to align operands.
operand_offset = INSTRUCTION_MAX_LEN - len(token[0])
operand_offset = ' ' * operand_offset
try:
# Instruction as a hexadecimal number.
encoded = bin_to_hex(parse(token), 8)
except Exception as exc:
if not flag_values['e']:
raise exc
print(exc)
encoded = '[invalid]'
# -l flag enables little-endian
if flag_values['l']:
encoded = hex_to_little_endian(encoded)
if not flag_values['s']:
instruction_info = token[0] + " " \
+ operand_offset \
+ ','.join(token[1:])
print(
dec_int_to_hex(address, 8) + " " + encoded +
" " + instruction_info)
else:
print(encoded)
address += 4
except Exception as exc:
print(exc)
except Exception as exc:
print("Error! Failed to open " + filename + "!")
exit(1)
def test_lexer_ident(self):
inputs = [
("x", [('ID', 'x'), 'EOF']),
("x123", [('ID', 'x123'), 'EOF']),
("x_123", [('ID', 'x_123'), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def __open_file(program):
try:
f = open(program.strip(), 'r')
except:
raise ValueError(
'Brainfuck: program {name} does not exist.'.format(name=program))
L = [line for line in f] # parse file
T = tokenize(L)
return T
def __main():
input_str = "(λx.x x)(λy.y)"
tokens = tokenize(input_str)
ast = parse(tokens)
from pprint import pprint
print(input_str)
pprint(tokens)
pprint(ast)
print(ast)
def test_lexer_fun_decl(self):
input_str = "void foo(long a, int b, char c) { }"
result = list(simplify(tokenize(input_str)))
expected = [('TYPE', 'void'), ('ID', 'foo'),
'LPAREN', ('TYPE', 'long'), ('ID', 'a'), 'COMMA',
('TYPE', 'int'), ('ID', 'b'), 'COMMA', ('TYPE', 'char'),
('ID', 'c'), 'RPAREN', 'LBRACE', 'RBRACE', 'EOF']
self.assertEqual(expected, result)
def test_comment(self):
expect = [
Token("NEWLINE", "\n", 1, 9),
Token("STRING", "not a comment", 2, 0),
]
tokens = tokenize("//comment\n\"not a comment\"")
for (token, ex) in itertools.izip(tokens, expect):
self.assertEqual(token.typ, ex.typ)
self.assertEqual(token.value, ex.value)
self.assertEqual(token.line, ex.line)
self.assertEqual(token.column, ex.column)
def test_lexer_while(self):
input_str = "while (x > 0) { x = 10; } "
result = list(simplify(tokenize(input_str)))
expected = ['WHILE', 'LPAREN', ('ID', 'x'),
'GREATER', ('NUMBER', 0), 'RPAREN',
'LBRACE', ('ID', 'x'), 'EQUAL', ('NUMBER', 10),
'SEMI', 'RBRACE', 'EOF']
self.assertEqual(expected, result)
def interpret(filename):
tokens = tuple(lexer.tokenize(filename))
defs, state = Parser(*tokens).parse()
gctx, errors = context_check(defs)
if errors:
for node, msg in errors:
print msg, "on line", node.line, "column", node.column
else:
return gctx["MAIN"].evaluate({}, gctx)
def __open_file(name):
# name is string without '.bf' suffix
file_path = name.strip() + '.bf'
try:
f = open(file_path, 'r')
except:
raise ValueError('Brainfuck: macro definition does not exist.')
L = [line for line in f] # parse file
T = tokenize(L)
f.close()
return T
def test_strings(self):
expect = [
Token("STRING", "I'm sorry, Dave", 1, 0),
Token("STRING", "Open the pod bay door", 1, 18),
]
tokens = tokenize("\"I'm sorry, Dave\" \"Open the pod bay door\"")
for (token, ex) in itertools.izip(tokens, expect):
self.assertEqual(token.typ, ex.typ)
self.assertEqual(token.value, ex.value)
self.assertEqual(token.line, ex.line)
self.assertEqual(token.column, ex.column)
def interpret(script, dictionary):
print(script)
tokens = lexer.tokenize(script)
tokens = list(tokens)
program = parser.parse_program(tokens, dictionary)
program, dictionary = evaluator.evaluate_program(program, dictionary)
return program, dictionary
def go(string): while True: tokens = lexer.tokenize(string) for t in tokens: print t, t.__class__ # entering loop print '>', # hold cursor in current line try: string = raw_input() except KeyboardInterrupt: # exit with ctrl+c return
def test_paren(self):
expect = [
Token("LPAREN", "(", 1, 0),
Token("ID", "a", 1, 1),
Token("RPAREN", ")", 1, 2),
]
tokens = tokenize("(a)")
for (token, ex) in itertools.izip(tokens, expect):
self.assertEqual(token.typ, ex.typ)
self.assertEqual(token.value, ex.value)
self.assertEqual(token.line, ex.line)
self.assertEqual(token.column, ex.column)
def test_lexer_compound_assign(self):
inputs = [
("x += 1", [('ID', 'x'), 'PLUS_EQ', ('NUMBER', 1), 'EOF']),
("x -= 1", [('ID', 'x'), 'MINUS_EQ', ('NUMBER', 1), 'EOF']),
("x *= 1", [('ID', 'x'), 'TIMES_EQ', ('NUMBER', 1), 'EOF']),
("x /= 1", [('ID', 'x'), 'DIVIDE_EQ', ('NUMBER', 1), 'EOF']),
("x %= 1", [('ID', 'x'), 'MOD_EQ', ('NUMBER', 1), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_assignment(self):
expect = [
Token("LET", "LET", 1, 0),
Token("ID", "a", 1, 4),
Token("ASSIGN", "BE", 1, 6),
Token("NUMBER", "25", 1, 9),
]
tokens = tokenize("LET a BE 25")
for (token, ex) in itertools.izip(tokens, expect):
self.assertEqual(token.typ, ex.typ)
self.assertEqual(token.value, ex.value)
self.assertEqual(token.line, ex.line)
self.assertEqual(token.column, ex.column)
def parser(input_string: str) -> None:
try:
token_list: TokenList = deque(tokenize(input_string))
parse(token_list)
print(f'{input_string} is a correct and valid expression')
except InvalidTokenException as e:
print(e)
except IncorrectTokenException as e:
print(e)
except ExcedingTokensException:
print(f'{input_string} is too long to be a valid expression')
except NoMoreTokensException:
print(f'{input_string} is to short to be a valid expression')
def test_if_else():
cases = [
('if 1 {}', [IfElse(Num(1), [], None)]),
('if 1 { return 0; }', [IfElse(Num(1), [Return(Num(0))], None)]),
('if 1 {} else {}', [IfElse(Num(1), [], [])]),
('if 1 {} else if 0 {}',
[IfElse(Num(1), [], [IfElse(Num(0), [], None)])]),
('if 1 {} else if 0 {} else {}',
[IfElse(Num(1), [], [IfElse(Num(0), [], [])])]),
]
for string, ast in cases:
tokens = lexer.tokenize(string)
assert bcparser.parse(tokens) == ast
def main(filename):
input = open(filename)
parser = Parser()
try:
tree = parser.parse(tokenize(input))
#print_tree(tree, parser.terminals)
run(tree)
except parser.ParseErrors, e:
for token, expected in e.errors:
print 'Found', token, 'when', expected, 'was expected'
def test_numbers(self):
expect = [
Token("NUMBER", "1", 1, 0),
Token("NUMBER", "2", 1, 2),
Token("NUMBER", "3.2", 1, 4),
Token("NUMBER", "0.1", 1, 8),
Token("NUMBER", "-5", 1, 12),
]
tokens = tokenize("1 2 3.2 0.1 -5")
for (token, ex) in itertools.izip(tokens, expect):
self.assertEqual(token.typ, ex.typ)
self.assertEqual(token.value, ex.value)
self.assertEqual(token.line, ex.line)
self.assertEqual(token.column, ex.column)
def uebb_compile(filename):
tokens = tuple(lexer.tokenize(filename))
defs, state = Parser(*tokens).parse()
gctx, errors = context_check(defs)
if errors:
for node, msg in errors:
print msg, "on line", node.line, "column", node.column
else:
fh = open(filename[:-3] + ".ma", "w")
for i in compile_program(gctx):
fh.write(repr(i) + "\n")
fh.close()
def test_escapes(self):
r"""Test tokenizing strings with escapes.
This is testing the string:
" \" \\ \n \\t "
without the spaces.
"""
self.assertEqual(lexer.tokenize(r'"\"\\\n\\t"', ""), [
Token(token_kinds.string,
[ord('"'),
ord("\\"),
ord("\n"),
ord("\\"),
ord("t"), 0])
])
def follow_includes(header_name: str) -> set:
path = get_standard_path(header_name)
if path is None:
return set()
if path not in header_to_symbol_cache:
print(header_name, path)
header_to_symbol_cache[path] = set()
with open(os.path.join(path)) as file:
tokens = list(lexer.tokenize(file.read()))
headers = get_headers(tokens)
for hdr in headers:
header_to_symbol_cache[path].update(follow_includes(hdr))
print(header_to_symbol_cache[path])
return header_to_symbol_cache[path]
def eval(self, text):
"""Evaluates the specified text within this interpreter's
environment.
line - a string
"""
tokens = lexer.tokenize(text)
if not tokens:
return None
tree = parser.parse(tokens)
if self.USE_TAIL_RECURSION:
analyzers.markTailContexts(tree)
return tree.eval(self.env)
def completer(self, input, state):
tokens = lexer.tokenize(input)
symbol = tokens[-1]
if not lexer.is_symbol(symbol):
return None
options = self.get_options(self.scope, symbol.name)
if state >= len(options):
return None
tokens[-1] = options[state]
return "".join(tokens)
def test(filename, expected_output):
print "### Interpreting", filename
interpreted_ast_output = interpreter.interpret(os.path.join(EXAMPLES, filename))
tokens = tuple(lexer.tokenize(os.path.join(EXAMPLES, filename)))
defs, state = Parser(*tokens).parse()
gctx, errors = context_check(defs)
print "### Testing compiled instructions:"
interpreted_instructions_output = uebb.interpret(coder.compile_program(gctx))
if interpreted_ast_output != expected_output:
raise RuntimeError("Interpreted output %d was incorrect, was expecting %d" % (interpreted_ast_output, expected_output))
if interpreted_instructions_output != expected_output:
raise RuntimeError("Compiled output %d was incorrect, was expecting %d" % (interpreted_instructions_output, expected_output))
print
print
def start(self):
stored_tokens = []
while True:
try:
line = raw_input("scheme> ").strip()
except EOFError:
print
break
if not line:
continue
try:
tokens = lexer.tokenize(line)
except Exception as e:
print(e)
continue
stored_tokens += tokens
ast, balance = lexer.get_ast(stored_tokens)
if balance > 0:
continue
elif balance < 0:
print('Unexpected ")"')
stored_tokens = []
continue
stored_tokens = []
ast = lexer.expand_quotes(ast)
ast = lexer.expand_define(ast)
for expr in ast:
print(self.scope.eval(expr))
elif code[i] == Opcode.HALT:
print addr(i) + " HALT"
else:
print addr(i) + " ?? " + str(code[i])
i += 1
if __name__ == "__main__":
if len(sys.argv) > 1:
print "opening file", sys.argv[1]
with open(sys.argv[1], 'r') as f:
prog = f.read()
else:
from samples import sample_prog as prog
ast = parse(tokenize(prog))
print "AST:"
pprint.pprint(ast)
(code, strings) = translate(ast)
#print "\nCode:"
#pprint.pprint(code)
print "\nStrings:"
pprint.pprint(strings)
print "\nDisassembly:"
disassemble(code)
import sys
from pprint import pprint
tokens = lexer.tokenize('''
(a=b)(c=d);
a = (b = c);
a(b(c));
a(b)(c)(d);
f(1,2,3, 4);
def a:int = 4;
b = 4 + a;
if 6-5 == 4 and a == b {
hallo;
}
else
{
hello;
a + b-c*2;
/* kommentar */
}
def foo(a:int, b:int):int {
return a + b;
}
7*8;
if 1 {
// kommentar
hello;}
''')
parser.init(tokens)
def test_basic_operators():
ops = ['+', '-', '*', '/', '^', '<', '>', '<=', '>=', '==', '!=']
for op in ops:
tokens = lexer.tokenize('1 {} 1'.format(op))
assert bcparser.parse(tokens) == [BinOp(Num(1), op, Num(1))]
def interpret(ins,mem,rpl_r=32,rpl_c=1,rpl_Q=10,rpl_W=0):
cmd = ins[0]
global program_input
global last_input
to_return = 0
if cmd == '[': # I only want for loops to pass through
newmem = Memory()
#newmem.instructions = copy.deepcopy(mem.instructions)
interpret_list(ins[1],newmem,_rpl_Q=rpl_Q,_rpl_W=rpl_W)
to_return = newmem.matrix
elif cmd == '{':
newmem = Memory()
#newmem.instructions = copy.deepcopy(mem.instructions)
newmem.matrix = copy.deepcopy(mem.matrix)
interpret_list(ins[1],newmem,_rpl_Q=rpl_Q,_rpl_W=rpl_W)
to_return = newmem.matrix
if len(ins)>2: del ins[2]
cpy = 0
if cmd in SPECIAL_COMS:
cpy = ins[1]
else:
cpy = [interpret(i,mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W) for i in ins[1]]
cpy = copy.deepcopy(cpy)
ins.append(cpy)
if cmd == '#':
to_return = ins[2][0]
elif cmd in MATHSET:
to_return= mm.apply_math(ins[2][0],ins[2][1],MATHDICT[cmd])
elif cmd in UNMATHSET:
to_return= mm.apply_unmath(ins[2][0],UNMATHDICT[cmd])
elif cmd in ADDSET:
mem.matrix = uu.add(mem.matrix,as_matrix(ins[2][0]),cmd)
to_return= mem.matrix
elif cmd in SHIFTSET:
mem.matrix = uu.shift(mem.matrix,as_float(ins[2][0]),cmd)
to_return= mem.matrix
elif cmd in OPSET:
mem.matrix = uu.apply_op(mem.matrix,cmd)
to_return= mem.matrix
elif cmd == 'r':
to_return= rpl_r
elif cmd == 'c':
to_return= rpl_c
elif cmd == 'Q':
to_return= rpl_Q
elif cmd == 'W':
to_return= rpl_W
elif cmd == 'L':
to_return = float(len(mem.matrix))
elif cmd == 'l':
to_return = float(len(mem.matrix[0]))
elif cmd == 'm':
rows = int(as_float(interpret(ins[2][1],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W)))
cols = int(as_float(interpret(ins[2][2],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W)))
mem.matrix = []
for r in range(rows):
row = []
for c in range(cols):
row.append(interpret(ins[2][0],mem,rpl_r=r,rpl_c=c,rpl_Q=rpl_Q,rpl_W=rpl_W))
mem.matrix.append(row)
to_return= mem.matrix
elif cmd == 'F':
rows = int(as_float(interpret(ins[2][1],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W)))
cols = int(as_float(interpret(ins[2][2],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W)))
for W in range(rows):
for Q in range(cols):
interpret(ins[2][0],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=Q,rpl_W=W)
to_return= 0
elif cmd == 'k':
mem.matrix = as_matrix(ins[2][0])
to_return= mem.matrix
elif cmd == 's':
_set(as_matrix(ins[2][0]),as_float(ins[2][1]),as_float(ins[2][2]),mem)
to_return= as_matrix(ins[2][0])
elif cmd == 'g':
to_return= _get(as_float(ins[2][0]),as_float(ins[2][1]),mem)
# jumps are unecesary for now. Just use for loops
# I plan to add them back in eventually,
# but for now it's just too big of a hassle
#elif cmd == 'j':
# to_return = 0
# _inslist = copy.deepcopy(mem.instructions[int(as_float(ins[2][0]))])
# _index = int(as_float(ins[2][1]))-1
elif cmd == 'i':
to_return = 0
if as_float(interpret(ins[2][0],mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W))!=0:
to_return = ins[2][1]
else:
to_return = ins[2][2]
to_return = interpret(to_return,mem,rpl_r=rpl_r,rpl_c=rpl_c,rpl_Q=rpl_Q,rpl_W=rpl_W)
#no use for this, just use ascii codes
#elif cmd == '"':
# pass
elif cmd == '\'':
print(chr(int(as_float(ins[2][0]))),end="")
to_return = 0
elif cmd == 'y':
if len(program_input)==0:
last_input = 0
to_return= 0
else:
last_input = ord(program_input[0])
program_input = program_input[1:]
to_return= last_input
elif cmd == 'n':
last_input = 0
if len(program_input)>0:
newinput = lexer.tokenize([c for c in program_input])
index=0
for num in newinput:
try:
nnum = float(num)
last_input = nnum
index = newinput.index(num)
break
except (TypeError, ValueError):
continue
if index>=len(newinput):
program_input=''
else:
program_input = program_input[program_input.index(newinput[index])+len(newinput[index]):]
to_return= last_input
elif cmd == 'p':
to_return= mm.matrix_product(as_matrix(ins[2][0]))
elif cmd == 'd':
to_return= mm.matrix_sum(as_matrix(ins[2][0]))
elif cmd == 'P':
to_return= mm.dot_product(as_matrix(ins[2][0]),as_matrix(ins[2][1]))
elif cmd == 'D':
to_return= mm.dot_sum(as_matrix(ins[2][0]),as_matrix(ins[2][1]))
elif cmd=='q':
mem.matrix=uu.cut_from_tl(mem.matrix,as_float(ins[2][0]),as_float(ins[2][1]))
to_return= mem.matrix
elif cmd=='z':
mem.matrix=uu.cut_from_br(mem.matrix,as_float(ins[2][0]),as_float(ins[2][1]))
to_return= mem.matrix
elif cmd == '?':
to_return= random.uniform(as_float(ins[2][0]),as_float(ins[2][1]))
elif cmd == '<':
ins[2] = [[interpret_list(col,mem,_rpl_Q=rpl_Q,_rpl_W=rpl_W) for col in row] for row in ins[2]]
to_return = uu.pad(ins[2])
elif cmd == 'N':
to_return = last_input
elif cmd == 'S':
print(str(ins[2][0]))
to_return = 0
elif cmd == 'C':
to_return = uu.contains(as_matrix(ins[2][0]), as_float(ins[2][1]))
return to_return
pprint.pprint({
"IP": self.IP,
"STACK": self.STACK,
"NAME_REG": self.NAME_REG,
"VARs": self.VARS,
"IP_STACK": self.IP_STACK,
"VAR_STACK": self.VAR_STACK,
})
def Run(self):
if(self.debugger):
self.PrintState()
while not self.halted:
self.Step()
if(self.debugger):
self.PrintState()
if __name__ == "__main__":
from samples import sample_prog
from lexer import tokenize
from parser import parse
from translator import translate
(code, strings) = translate(parse(tokenize(sample_prog)))
vm = BasicVM()
vm.Load(code, strings)
#vm.SetDebugger(True)
vm.Run()
def lex_into_list(str): return [token for token in lexer.tokenize(str)]
return PExpr(expr)
def parse(tokens):
return Parser(tokens).parse()
if __name__ == "__main__":
if len(sys.argv) > 1:
print "opening file", sys.argv[1]
from lexer import tokenize
with open(sys.argv[1], "r") as f:
statements = f.read()
ast = parse(tokenize(statements))
else:
print "using baked-in file"
ast = parse(
[
Token("CLEAR", "CLEAR", 1, 0),
Token("ID", "top", 3, 0),
Token("COLON", ":", 3, 3),
Token("NEWLINE", "\n", 3, 4),
Token("LET", "LET", 4, 0),
Token("ID", "a", 4, 4),
Token("ASSIGN", "BE", 4, 6),
Token("NUMBER", "25", 4, 9),
Token("NEWLINE", "\n", 4, 11),
Token("PRINT", "PRINT", 5, 0),
Token("STRING", "Hello world", 5, 6),
def pop(state, *token_classes):
"""
Check whether the head of tokens is instance of any of the required classes, return it if it is otherwise throw exception
"""
for token_class in token_classes:
if isinstance(state.head, token_class):
return state.head, Parser(*state.tail)
# No token class was matched, raise exception.
# Note that as parsing is very strict it does not make much sense to
# gather all error messages because the subsequent ones don't make much sense.
raise ParseError("Got " + repr(state.head) + " of class " + state.head.__class__.__name__ + ", was expecting " + " or ".join([tc.__name__ for tc in token_classes]), state.head)
def peek(state, token_class):
return isinstance(state.head, token_class)
def skip(state, *token_classes):
token, state = state.pop(*token_classes)
return state
if __name__ == "__main__":
filename, = sys.argv[1:]
tokens = tuple(lexer.tokenize(filename))
defs, state = Parser(*tokens).parse()
print "Parsed defs:"
for d in defs:
print d
import pretty_printer import sys import lexer import parser #import code_generator #import interpreter filename = sys.argv[1] tokens = lexer.tokenize(filename) #print tokens goal = parser.parse(tokens) pretty_printer.pretty_print(goal) code_generator.generate(goal) assembler.assemble(asm_file) linker.link(o_file)
