class TestLexer(unittest.TestCase): def setUp(self): self.lex = Lexer("Test/Examples/gcd.oats") self.expected_tokens = [ Token("special", "func", 1), Token("word", "gcd", 1), Token("special", "(", 1), Token("word", "x", 1), Token("special", ",", 1), Token("word", "y", 1), Token("special", ")", 1), Token("special", "{", 1), ] def tearDown(self): self.lex.f.close() def test_next_token(self): ct = self.lex.next_token() def checkEquivalent(token1, token2): self.assertEqual(token1.type_, token2.type_) self.assertEqual(token1.lexeme, token2.lexeme) for k in range(len(self.expected_tokens)): checkEquivalent(self.expected_tokens[k], ct) ct = self.lex.next_token() def test_prev_token(self): ct = self.lex.next_token() def checkEquivalent(token1, token2): self.assertEqual(token1.type_, token2.type_) self.assertEqual(token1.lexeme, token2.lexeme) for k in range(len(self.expected_tokens)): ct = self.lex.next_token() for k in reversed(range(len(self.expected_tokens))): ct = self.lex.prev_token() checkEquivalent(self.expected_tokens[k], ct)
def test_double_char_operators(self): string = '<= > <= < < >= = < > >= < <= <= < >= <= >= >= != > >=' tokens = [ Token(TokenType.LEQ), Token(TokenType.GRE), Token(TokenType.LEQ), Token(TokenType.LESS), Token(TokenType.LESS), Token(TokenType.GEQ), Token(TokenType.ASSIGN), Token(TokenType.LESS), Token(TokenType.GRE), Token(TokenType.GEQ), Token(TokenType.LESS), Token(TokenType.LEQ), Token(TokenType.LEQ), Token(TokenType.LESS), Token(TokenType.GEQ), Token(TokenType.LEQ), Token(TokenType.GEQ), Token(TokenType.GEQ), Token(TokenType.NEQ), Token(TokenType.GRE), Token(TokenType.GEQ) ] lexer = Lexer(StringSource(string)) for expected_token in tokens: token = lexer.current_token self.assertEqual(expected_token.type, token.type) self.assertEqual(expected_token.value, token.value) lexer.build_next_token()
class Parser: def __init__(self, source): self.lexer = Lexer(source) self.source = source self.start_of_object_pos = 0 def expect(self, expected_token_type): if self.lexer.current_token.type != expected_token_type: self.error(error_code=ErrorCode.UNEXPECTED_TOKEN, expected=expected_token_type) prev_token = self.lexer.current_token self.lexer.build_next_token() return prev_token def expect_not_none(self, tested_object, error_description=''): if tested_object is None: self.error(error_code=ErrorCode.EXPECTED_NOT_NONE, description=error_description) return tested_object def parse_program(self): toplevel_objects = [] while (parsed_object := self.try_to_parse_fun_definition()) or\ (parsed_object := self.try_to_parse_statement()):
def setUp(self): self.lex = Lexer("Test/Examples/gcd.oats") self.expected_tokens = [ Token("special", "func", 1), Token("word", "gcd", 1), Token("special", "(", 1), Token("word", "x", 1), Token("special", ",", 1), Token("word", "y", 1), Token("special", ")", 1), Token("special", "{", 1), ]
def test_trivial(self): file_source = FileSource('tokens/positions1.txt') lexer = Lexer(file_source) # positions within line positions = [1, 2, 3, 5, 9, 11, 14, 16] for expected_position in positions: token = lexer.current_token self.assertEqual(expected_position, token.position.column) lexer.build_next_token()
def test_scalar_starts_with_zeros(self): string = '0' # should pass try: Lexer(StringSource(string)) except LexerError: self.fail() string += '0' # should fail with self.assertRaises(LexerError): Lexer(StringSource(string))
def test_too_long_id(self): id = ''.join(['a'] * 128) # should pass try: Lexer(StringSource(id)) except LexerError: self.fail() # should fail id += 'a' with self.assertRaises(LexerError) as e: Lexer(StringSource(id)) self.assertEqual(ErrorCode.EXCEED_MAX_ID_SIZE, e.exception.error_code)
def test_all_tokens(self): tokens = [Token(t) for t in RESERVED_KEYWORDS.values()] + [ Token(TokenType.ID, 'a'), Token(TokenType.ID, 'aaa'), Token(TokenType.ID, 'a123'), Token(TokenType.ID, 'a_'), Token(TokenType.ID, 'a_123'), Token(TokenType.ID, 'abc_def_123gh'), Token(TokenType.SEMI), Token(TokenType.COMMA), Token(TokenType.COLON), Token(TokenType.PLUS), Token(TokenType.MINUS), Token(TokenType.MUL), Token(TokenType.FLOAT_DIV), Token(TokenType.ASSIGN), Token(TokenType.LPAREN), Token(TokenType.RPAREN), Token(TokenType.LBRACK), Token(TokenType.RBRACK), Token(TokenType.LCURB), Token(TokenType.RCURB), Token(TokenType.LESS), Token(TokenType.GRE), Token(TokenType.LEQ), Token(TokenType.GEQ), Token(TokenType.EQ), Token(TokenType.NEQ), Token(TokenType.POW), Token(TokenType.SCALAR, 0), Token(TokenType.SCALAR, 12), Token(TokenType.SCALAR, 12.345), Token(TokenType.SCALAR, 12.345), Token(TokenType.SCALAR, float('12.345e6')), Token(TokenType.SCALAR, float('12.345e-6')), Token(TokenType.SCALAR, 0), Token(TokenType.SCALAR, 0.01), Token(TokenType.SCALAR, float('0.001e2')), Token(TokenType.SCALAR, float('0.0001e-2')), Token(TokenType.ETX) ] file_source = FileSource('tokens/all_tokens.txt') lexer = Lexer(file_source) for expected_token in tokens: token = lexer.current_token self.assertEqual(expected_token.type, token.type) self.assertEqual(expected_token.value, token.value) lexer.build_next_token()
def main(filename): print("----------Parser Debug-------------") tokenizer = Tokenizer(filename) tokenizer.tokenize() lexer = Lexer(tokenizer.getTokens()) lexer.lex() parser = Parser(lexer.getTokens()) parser.parse() print(parser.getTree()) print("\n----------Execution Stack-----------") interpreter = Interpreter(parser.getTree()) interpreter.interpret() print("\n----------Program Output------------") return interpreter.output()
def test_real_life_problems(self): file_source = FileSource('tokens/positions3.txt') lexer = Lexer(file_source) positions = [ (2, 1), (2, 5), (2, 20), (2, 21), (2, 22), (3, 5), (3, 13), (3, 15), (3, 31), (3, 33), (3, 49), (3, 51), (3, 52), (4, 1) ] for expected_position in positions: token = lexer.current_token self.assertEqual(expected_position[0], token.position.line) self.assertEqual(expected_position[1], token.position.column) lexer.build_next_token()
def test_unknown_single_char_token(self): string = '^' string_source = StringSource(string) with self.assertRaises(LexerError) as e: Lexer(string_source) self.assertNotEqual(ErrorCode.UNEXPECTED_TOKEN, e.exception.error_code) self.assertEqual(ErrorCode.TOKEN_BUILD_FAIL, e.exception.error_code)
def testLexComments(self): tokens = [] tokens.append(Token(0, "#", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "#", None)) self.lexer = Lexer(tokens) self.lexer.lex() tokens = [] tokens.append(Token(0, "#", commentType())) tokens.append(Token(1, ")", commentType())) tokens.append(Token(2, "+", commentType())) tokens.append(Token(3, "-", commentType())) tokens.append(Token(4, "*", commentType())) tokens.append(Token(5, "/", commentType())) tokens.append(Token(6, "#", commentType())) for token1, token2 in zip(self.lexer.tokens, tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect")
def testGetTokensPostLex(self): tokens = [] tokens.append(Token(0, "(", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "11", None)) self.lexer = Lexer(tokens) self.lexer.lex() tokens = [] tokens.append(Token(0, "(", statementBeginType())) tokens.append(Token(1, ")", statementEndType())) tokens.append(Token(2, "+", addOpType())) tokens.append(Token(3, "-", subOpType())) tokens.append(Token(4, "*", multOpType())) tokens.append(Token(5, "/", divOpType())) tokens.append(Token(6, 11, numType())) for token1, token2 in zip(self.lexer.getTokens(), tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect")
def test_etx_on_comment_line(self): string = 'not_comment = 1; # a comment' lexer = Lexer(StringSource(string)) tokens = [ Token(TokenType.ID, 'not_comment'), Token(TokenType.ASSIGN), Token(TokenType.SCALAR, 1), Token(TokenType.SEMI), Token(TokenType.ETX) ] for expected_token in tokens: token = lexer.current_token self.assertEqual(expected_token.type, token.type) self.assertEqual(expected_token.value, token.value) lexer.build_next_token()
def testInit(self): tokens = [] tokens.append(Token(0, "a", None)) tokens.append(Token(1, "line", None)) tokens.append(Token(2, "the", None)) tokens.append(Token(3, "four", None)) self.lexer = Lexer(tokens) self.assertEqual(self.lexer.tokens, tokens, "Initialization not correct")
def testInterpreterFib(self, mock_input, mock_output): """Verify the correctness of the fibonnaci.oats program (The 20th fibonnaci number is 6765).""" mock_input.side_effect = ["20"] lex = Lexer("Test/Examples/fibonnaci.oats") sparser = StatementParser(lex) root = sparser.parse_main() sym_tab = [dict()] sinterpreter = StatementInterpreter(sym_tab) sinterpreter.interpret_statement_list(root) self.assertEqual(mock_output.getvalue(), "6765\n")
def testInterpreterGCD(self, mock_input, mock_output): """Verify the correctness of the gcd.oats program (The gcd of 1071 and 462 is 21).""" mock_input.side_effect = ["1071", "462"] lex = Lexer("Test/Examples/gcd.oats") sparser = StatementParser(lex) root = sparser.parse_main() sym_tab = [dict()] sinterpreter = StatementInterpreter(sym_tab) sinterpreter.interpret_statement_list(root) self.assertEqual(mock_output.getvalue(), "21\n")
def test_positions_on_multiple_lines(self): file_source = FileSource('tokens/positions2.txt') lexer = Lexer(file_source) positions = [ (2, 1), (2, 3), (2, 5), (2, 8), (3, 1), (3, 3), (3, 5), (3, 7), (3, 8), (3, 9), (4, 1), (5, 5), (5, 6), (6, 10), (6, 12), (7, 7), (7, 9), (8, 1), (8, 4), (8, 6), (8, 10), (8, 18), (8, 19), (8, 21) ] for expected_position in positions: token = lexer.current_token self.assertEqual(expected_position[0], token.position.line) self.assertEqual(expected_position[1], token.position.column) lexer.build_next_token()
def testGetTokensPreLex(self): tokens = [] tokens.append(Token(0, "#", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "#", None)) self.lexer = Lexer(tokens) for token1, token2 in zip(self.lexer.getTokens(), tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect")
def setUp(self): self.lex = Lexer("Test/Examples/gcd.oats") self.sparser = StatementParser(self.lex) self.expected_nodes = [ Node(Node.STATEMENT_LIST_TYPE), Node(Node.FUNC_DEF_TYPE, "gcd"), Node(Node.PARAMETER_LIST_TYPE), Node(Node.VARIABLE_TYPE, "x"), Node(Node.VARIABLE_TYPE, "y"), Node(Node.STATEMENT_LIST_TYPE), Node(Node.IF_TYPE), Node(Node.LT_TYPE), Node(Node.VARIABLE_TYPE, "x"), Node(Node.VARIABLE_TYPE, "y"), Node(Node.STATEMENT_LIST_TYPE), Node(Node.ASSIGN_TYPE), ]
def empezar(): import sys archivoEntrada = open(sys.argv[1], 'r') analizadorLexico = Lexer() entrada = archivoEntrada.read() analizadorLexico.input(entrada) lineatokens = '' try: for token in iter(analizadorLexico.lexer.token, None): lineatokens += repr(token.type) + ' ' + repr( token.value) + ' ' + repr(token.lineno) lineatokens += '\n' except Error.LexicalError.LexicalError as error: lineatokens = error.mensaje archivoEntrada.close() analizadorSintactico = Parser(analizadorLexico.tokens) analizadorLexico.lexer.lineno = 1 try: raiz = analizadorSintactico.parse(entrada) Parser.linea += 'Programa:\n' for clase in raiz: clase.imprimir() except Error.SyntacticalError.SyntacticalError as error: Parser.linea = error.mensaje analizadorSemantico = VisitanteTabla() try: analizadorSemantico.visitarProgram(raiz) except Error.SemanticError.SemanticError as error: try: analizadorSemantico.linea = error.mensaje analizadorSemantico.linea += '\nEl error se dio en la clase: ' + analizadorSemantico.metodoActual.tabla.padre.nombre + ', metodo: ' + analizadorSemantico.metodoActual.tabla.nombre except BaseException: pass analizadorCodigo = VisitanteLir(analizadorSemantico.tablaPrincipal) analizadorCodigo.visitarProgram(raiz) dump_tokens = 0 dump_ast = 0 dump_symtab = 0 dump_lir = 0 for parametro in sys.argv: if parametro[0] == '-': if parametro == '-dump-tokens': dump_tokens = 1 if parametro == '-dump-ast': dump_ast = 1 if parametro == '-dump-symtab': dump_symtab = 1 if parametro == '-dump-lir': dump_lir = 1 if (dump_tokens): archivoSalida = open(sys.argv[1][:-3] + '.tok', 'w') archivoSalida.write(lineatokens) print "Tokens fueron escritos a: %s" % (str(sys.argv[1][:-3] + '.tok')) archivoSalida.close() if (dump_ast): archivoSalida = open(sys.argv[1][:-3] + '.ast', 'w') archivoSalida.write(Parser.linea) print "Salida del AST fue escrita a: %s" % (str(sys.argv[1][:-3] + '.ast')) archivoSalida.close() if (dump_symtab): archivoSalida = open(sys.argv[1][:-3] + '.sym', 'w') archivoSalida.write(analizadorSemantico.linea) print "Salida de la tabla de simbolos fue escrita a: %s" % ( str(sys.argv[1][:-3] + '.sym')) archivoSalida.close() if (dump_lir): archivoSalida = open(sys.argv[1][:-3] + '.lir', 'w') archivoSalida.write(analizadorCodigo.linea) print "Codigo intermedio escrito a: %s" % (str(sys.argv[1][:-3] + '.lir')) archivoSalida.close() listabat = analizadorCodigo.linea.split('\n') lineabat = '@ECHO OFF\n' if listabat[0] == '': listabat = listabat[1:] lineabat += 'echo ' + listabat[0] + '>codigo.tmp\n' for linea in listabat[1:]: if linea == '': lineabat += 'echo+>>codigo.tmp\n' else: lineabat += 'echo ' + linea + '>>codigo.tmp\n' lineabat += 'java -jar microLIR.jar codigo.tmp\necho+\npause\ndel codigo.tmp' archivoSalida = open(sys.argv[1][:-3] + '.bat', 'w') archivoSalida.write(lineabat) print 'Archivo batch ejecutable escrito a: ' + sys.argv[1][:-3] + '.bat' archivoSalida.close()
def test_etx_in_unfinished_string(self): string = '"some random string' with self.assertRaises(LexerError): lexer = Lexer(StringSource(string)) while lexer.current_token.type != TokenType.ETX: lexer.build_next_token()
from Lexer.Lexer import Lexer from Parser.StatementParser import StatementParser from Interpreter.StatementInterpreter import StatementInterpreter from Common.Common import * import argparse parser = argparse.ArgumentParser() parser.add_argument("file", nargs=1, help="Filename of the .oats file to be interpreted") parser.add_argument("-d", "--debug", action='store_true') args = parser.parse_args() lexer = Lexer(args.file[0]) parser = StatementParser(lexer) root = parser.parse_main() if args.debug: print("-" * 50) print("PARSE TREE") print("-" * 50) root.print_recursive() print("-" * 50) print("PROGRAM") print("-" * 50) sym_tab = [dict()] interpreter = StatementInterpreter(sym_tab) interpreter.interpret_statement_list(root)
def empezar(): import sys archivoEntrada = open(sys.argv[1], 'r') analizadorLexico = Lexer() entrada = archivoEntrada.read() analizadorLexico.input(entrada) lineatokens = '' try: for token in iter(analizadorLexico.lexer.token, None): lineatokens += repr(token.type) + ' ' + repr(token.value) + ' ' + repr(token.lineno) lineatokens += '\n' except Error.LexicalError.LexicalError as error: lineatokens = error.mensaje archivoEntrada.close() analizadorSintactico = Parser(analizadorLexico.tokens) analizadorLexico.lexer.lineno = 1 try: raiz = analizadorSintactico.parse(entrada) Parser.linea += 'Programa:\n' for clase in raiz: clase.imprimir() except Error.SyntacticalError.SyntacticalError as error: Parser.linea = error.mensaje analizadorSemantico = VisitanteTabla() try: analizadorSemantico.visitarProgram(raiz) except Error.SemanticError.SemanticError as error: try: analizadorSemantico.linea = error.mensaje analizadorSemantico.linea += '\nEl error se dio en la clase: ' + analizadorSemantico.metodoActual.tabla.padre.nombre + ', metodo: ' + analizadorSemantico.metodoActual.tabla.nombre except BaseException: pass analizadorCodigo = VisitanteLir(analizadorSemantico.tablaPrincipal) analizadorCodigo.visitarProgram(raiz) dump_tokens = 0 dump_ast = 0 dump_symtab = 0 dump_lir = 0 for parametro in sys.argv: if parametro[0] == '-': if parametro == '-dump-tokens': dump_tokens = 1 if parametro == '-dump-ast': dump_ast = 1 if parametro == '-dump-symtab': dump_symtab = 1 if parametro == '-dump-lir': dump_lir = 1 if(dump_tokens): archivoSalida = open(sys.argv[1][:-3] + '.tok', 'w') archivoSalida.write(lineatokens) print "Tokens fueron escritos a: %s" % (str(sys.argv[1][:-3] + '.tok')) archivoSalida.close() if(dump_ast): archivoSalida = open(sys.argv[1][:-3] + '.ast', 'w') archivoSalida.write(Parser.linea) print "Salida del AST fue escrita a: %s" % (str(sys.argv[1][:-3] + '.ast')) archivoSalida.close() if(dump_symtab): archivoSalida = open(sys.argv[1][:-3] + '.sym', 'w') archivoSalida.write(analizadorSemantico.linea) print "Salida de la tabla de simbolos fue escrita a: %s" % (str(sys.argv[1][:-3] + '.sym')) archivoSalida.close() if(dump_lir): archivoSalida = open(sys.argv[1][:-3] + '.lir', 'w') archivoSalida.write(analizadorCodigo.linea) print "Codigo intermedio escrito a: %s" % (str(sys.argv[1][:-3] + '.lir')) archivoSalida.close() listabat = analizadorCodigo.linea.split('\n') lineabat = '@ECHO OFF\n' if listabat[0] == '': listabat = listabat[1:] lineabat += 'echo ' + listabat[0] + '>codigo.tmp\n' for linea in listabat[1:]: if linea == '': lineabat += 'echo+>>codigo.tmp\n' else: lineabat += 'echo ' + linea + '>>codigo.tmp\n' lineabat += 'java -jar microLIR.jar codigo.tmp\necho+\npause\ndel codigo.tmp' archivoSalida = open(sys.argv[1][:-3] + '.bat', 'w') archivoSalida.write(lineabat) print 'Archivo batch ejecutable escrito a: ' + sys.argv[1][:-3] + '.bat' archivoSalida.close()
import sys from Lexer.Lexer import Lexer from Parser.Parser import Parser from Interpreter.Interpreter import Interpreter from Lexer.LexerHash import LexerHash from Lexer.LexerQueue import LexerQueue Lexer.run(sys.argv) Parser.run() Interpreter.run() for arg in sys.argv: if arg == "-v": print("Tabela de variáveis") LexerHash.shared().verbose() print("\nLista de instruções") LexerQueue.shared().verbose()
def setUp(self): self.lexer = Lexer(None)
class testLexer(unittest.TestCase): def setUp(self): self.lexer = Lexer(None) def testInit(self): tokens = [] tokens.append(Token(0, "a", None)) tokens.append(Token(1, "line", None)) tokens.append(Token(2, "the", None)) tokens.append(Token(3, "four", None)) self.lexer = Lexer(tokens) self.assertEqual(self.lexer.tokens, tokens, "Initialization not correct") def testLexNoComments(self): tokens = [] tokens.append(Token(0, "(", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "11", None)) self.lexer = Lexer(tokens) self.lexer.lex() tokens = [] tokens.append(Token(0, "(", statementBeginType())) tokens.append(Token(1, ")", statementEndType())) tokens.append(Token(2, "+", addOpType())) tokens.append(Token(3, "-", subOpType())) tokens.append(Token(4, "*", multOpType())) tokens.append(Token(5, "/", divOpType())) tokens.append(Token(6, 11, numType())) for token1, token2 in zip(self.lexer.tokens, tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect") def testLexComments(self): tokens = [] tokens.append(Token(0, "#", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "#", None)) self.lexer = Lexer(tokens) self.lexer.lex() tokens = [] tokens.append(Token(0, "#", commentType())) tokens.append(Token(1, ")", commentType())) tokens.append(Token(2, "+", commentType())) tokens.append(Token(3, "-", commentType())) tokens.append(Token(4, "*", commentType())) tokens.append(Token(5, "/", commentType())) tokens.append(Token(6, "#", commentType())) for token1, token2 in zip(self.lexer.tokens, tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect") def testGetTokensPreLex(self): tokens = [] tokens.append(Token(0, "#", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "#", None)) self.lexer = Lexer(tokens) for token1, token2 in zip(self.lexer.getTokens(), tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect") def testGetTokensPostLex(self): tokens = [] tokens.append(Token(0, "(", None)) tokens.append(Token(1, ")", None)) tokens.append(Token(2, "+", None)) tokens.append(Token(3, "-", None)) tokens.append(Token(4, "*", None)) tokens.append(Token(5, "/", None)) tokens.append(Token(6, "11", None)) self.lexer = Lexer(tokens) self.lexer.lex() tokens = [] tokens.append(Token(0, "(", statementBeginType())) tokens.append(Token(1, ")", statementEndType())) tokens.append(Token(2, "+", addOpType())) tokens.append(Token(3, "-", subOpType())) tokens.append(Token(4, "*", multOpType())) tokens.append(Token(5, "/", divOpType())) tokens.append(Token(6, 11, numType())) for token1, token2 in zip(self.lexer.getTokens(), tokens): self.assertEqual(token1.id, token2.id, "Tokenization incorect") self.assertEqual(token1.literal, token2.literal, "Tokenization incorect") self.assertEqual(type(token1.type), type(token2.type), "Tokenization incorect")
def __init__(self, source): self.lexer = Lexer(source) self.source = source self.start_of_object_pos = 0