def test_accepted_init_lexer(source_code, file_name): l = None if source_code == "" and file_name == "": l = lexer.Lexer() file_name = "CLI" elif source_code == "": l = lexer.Lexer(filename=file_name) elif file_name == "": l = lexer.Lexer(source_code=source_code) file_name = "CLI" else: l = lexer.Lexer(source_code, file_name) assert isinstance(l, (lexer.Lexer)) and (l.error is None) and \ (l.filename == file_name) and (l.source_code == source_code)
def main(filename): try: # open the file for lexical analysis file_stream = open(filename, 'r') the_lexer = lexer.Lexer(file_stream) t = the_lexer.next_token() # while not at end of file, while t != None and t.tokentype != token.EOS: # print the next token print t t = the_lexer.next_token() # in case the user chooses a bad file, let them know why it's not working except IOError as e: print "error: unable to open file '" + filename + "'" sys.exit(1) except error.Error as e: print e sys.exit(1) if __name__ == '__main__': if len(sys.argv) != 2: print 'usage:', sys.argv[0], 'source-code-file' sys.exit(1) else: main(sys.argv[1])
def __init__(self): lex_rules = ( ('if', 'IF'), ('then', 'THEN'), ('else', 'ELSE'), ('true', 'TRUE'), ('false', 'FALSE'), ('lambda', 'LAMBDA'), ('\d+', 'INT'), ('->', 'ARROW'), ('!=', '!='), ('==', '=='), ('>=', '>='), ('<=', '<='), ('<', '<'), ('>', '>'), ('\+', '+'), ('\-', '-'), ('\*', '*'), ('\(', '('), ('\)', ')'), ('=', '='), (',', ','), ('[a-zA-Z_]\w*', 'ID'), ) self.lexer = lexer.Lexer(lex_rules, skip_whitespace=True) self.cur_token = None self.operators = {'!=', '==', '>=', '<=', '<', '>', '+', '-', '*'}
def __init__(self, rules): # initialize the lex rules for the grammar self.lx = lexer.Lexer(rules, skip_whitespace=True) self.cur_token = None self.next_token = None # dictionary for variables (e.g. set x = 10) self.vars = {}
def run(text): # Generate tokens # print("RUN1") lexer = lex.Lexer(text) tokens, error = lexer.make_tokens() if error: print() # return None, error # Generate AST print("RUN2 | Токены: ", tokens) parser = pars.Parser(tokens) ast = parser.parse() # print(ast.node) # if ast.error: # print() # return (ast.error) # Run program print("RUN3 | АСТ:", ast.node) interpreter = inter.Interpreter() context = inter.Context('<program>') context.symbol_table = global_symbol_table result = interpreter.visit(ast.node, context) print(result.value) return result.error
def __init__(self, **kwargs): self.lexer = lexer.Lexer(**kwargs) self.tokens = self.lexer.tokens self.parser = yacc(debug=False, debuglog=None, module=self, outputdir=tempfile.gettempdir())
def __init__(self, args=None): """ Get the options from cli. """ self.cli = cli.CLI() self.args = self.cli.parse_arguments(args) self.lexer = lexer.Lexer()
def TestReturnStatements(self): print("Running TestReturnStatements ...") @dataclass class test: input: str expectedValue: object tests = [ test("return 5", 5), test("return true", True), test("return y", "y"), ] for tt in tests: l = lxr.Lexer(tt.input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if program is None: raise Exception('ParseProgram() returned None') if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not self.testReturnStatement(stmt): return val = stmt.Value if not self.testLiteralExpression(val, tt.expectedValue): return
def TestFunctionParameterParsing(self): @dataclass class test: input: str expectedParams: List[str] tests = [ test("fn() {}",[]), test("fn(x) {}",['x']), test("fn(x, y, z) {}",['x', 'y', 'z']) ] for tt in tests: l = lxr.Lexer(tt.input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') function = stmt.Expression if len(function.Parameters) != len(tt.expectedParams): raise Exception(f'function.Parameters does not contain {len(tt.expectedParams)} parameters got {len(function.Parameters)}') ix = 0 for p in tt.expectedParams: self.testLiteralExpression(function.Parameters[ix], p) ix += 1
def TestCallExpressionParsing(self): print("Running TestCallExpressionParsing ...") input = """ add(1, 2 * 3, 4 + 5) """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statement got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') funcCall = stmt.Expression if not isinstance(funcCall, ast.CallExpression): raise Exception(f'stmt.Expression is not a ast.CallExpression got={stmt.Expression}') if not self.testIdentifier(funcCall.Function, "add"): return if len(function.Arguments) != 3: raise Exception(f'function.Arguments does not contain 3 statements got={stmt.Expression}') self.testLiteralExpression(function.Arguments[0],1) self.testInfixExpression(function.Arguments[1], 2, "*", 3) self.testInfixExpression(function.Arguments[2], 4, "+", 5)
def TestIntegerLiteralExpressions(self): print("Running TestIntegerLiteralExpressions ...") input = """ 5 """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') literal = stmt.Expression if not isinstance(literal, ast.IntegerLiteral): raise Exception(f'Expression is not a ast.Identifier got={stmt.Expression}') if literal.Value != 5: raise Exception(f'ident.Value is not 5 got={literal.Value}') if literal.TokenLiteral() != "5": raise Exception(f'ident.TokenLiteral() is not "5" got={literal.TokenLiteral()}')
def TestBooleanExpressions(self): print("Running TestBooleanExpressions ...") input = """ true """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') ident = stmt.Expression if not isinstance(ident, ast.Boolean): raise Exception(f'Expression is not a ast.Boolean got={stmt.Expression}') if ident.Value != True: raise Exception(f'ident.Value is not True got={ident.Value}') if ident.TokenLiteral() != "true": raise Exception(f'ident.TokenLiteral() is not True got={ident.TokenLiteral()}')
def __init__(self, filename): lex = lexer.Lexer() self.tokens = lex.token_generator(filename) # currtok is the current token that we are looking at self.currtok = next(self.tokens)
def main(): # Variable to store the contents of the file content = "" with open('test.lang', 'r') as file: # reading and storing the contents of the file content = file.read() ############################ # Lexer # ############################ # Calling the lexer file Lexer class and initializing it with the source code in lex lex = lexer.Lexer(content) # Calling the tokenize method from inside the lexer instance tokens = lex.tokenize() ############################ # Parser # ############################ # Calling the parser file Parser class and initializing it with the source code parse = parser.Par(tokens) # Calling the parse method from inside the parse instance parse.parse()
def __init__(self, compiler='aspCompiler', silent=False, decoupled=False, agent=False): self.silent = silent self.debug = False self.ignore_errors = False self.ignore_undefined = False self.there_were_roles = False self.roles = [] self.lex = lexer.Lexer() self.lex.build() self.par = parser.Parser() self.par.build() if compiler == 'aspCompiler': self.comp = aspCompiler.AspCompiler(decoupled=decoupled, agent=agent) elif compiler == 'aspReducedCompiler': self.comp = aspReducedCompiler.AspReducedCompiler( decoupled=decoupled, agent=agent) else: if not self.silent: print >> sys.stderr, "Could not find Compiler ", compiler, ". Sticking to aspCompiler." self.comp = aspCompiler.AspCompiler(decoupled=decoupled, agent=agent) self.default_output_path = "./" #"../temp/"
def eval_test(input: str) -> MonkeyObject: l = lexer.Lexer(input) p = parser.Parser(l) program = p.parse_program() env = mobject.Environment() return evaluator.eval(program, env)
def __init__(self): lex_rules = ( ("if", "IF"), ("then", "THEN"), ("else", "ELSE"), ("true", "TRUE"), ("false", "FALSE"), ("\d+", "INT"), ("->", "ARROW"), ("!=", "NE"), ("==", "EQ"), (">=", "GE"), (">", "GT"), ("<=", "LE"), ("<", "LT"), ("\+", "PLUS"), ("\-", "MINUS"), ("\*", "MUL"), ("\/", "DIV")("\%", "MOD"), ("\(", "LPAREN"), ("\)", "RPAREN"), ("=", "ASSIGN"), (",", "DOT"), ("[a-zA-Z_]\w*", "ID"), ) self.lexer = lexer.Lexer(lex_rules, skip_whitesapce=True) self.cur_token = None self.operators = {"!=", "==", ">=", "<=", "<", ">", "+", "-", "*", "%"}
def parse_imports(self, parent_unit: ast.TranslationUnit): for imp_stmt in parent_unit.import_statements: if imp_stmt.path not in self.modules: # Parse # TODO this should be encapsulated more nicely. Currently, same code # as in main.py try: with open(imp_stmt.path) as f: code = f.read() except Exception as e: raise TypecheckException( f"Importing {imp_stmt.path} impossible:" f" '{e}'", imp_stmt) l = lexer.Lexer(code, imp_stmt.path) p = parser.Parser(l) child_unit = p.compile() # add2modmap if not isinstance(child_unit, ast.TranslationUnit): raise TypecheckException( f"Importing {imp_stmt.path}" " failed.", imp_stmt) self.modules[imp_stmt.path] = child_unit # Recurse self.parse_imports(child_unit) # Add to symbol table parent_unit.symbols_global[imp_stmt.name] = bongtypes.Module( imp_stmt.path)
def __init__(self, stmt_as_is=False): self.lexer = lexer.Lexer(stmt_as_is) # TODO: `write_tables=0` is a temporary workaround until we find a directory to settle in self.parser = yacc.yacc(start="program", debug=True, debuglog=logging.getLogger('parser'), errorlog=logging.getLogger('parser2'), write_tables=0)
def doMain(args): # Parse all legacy files. import seqan.dddoc.core as core app = core.App() for path in args.legacy_doc_dirs: print 'Scanning %s...' % path app.loadFiles(path) migrated_doc = raw_doc.RawDoc() if args.legacy_doc_dirs: app.loadingComplete() migrated_doc.entries = migration.migrate(app.dddoc_tree) print 'migrated_doc.entries', [e.name.text for e in migrated_doc.entries] # Parse all normal input files. fmgr = file_mgr.FileManager() master_doc = raw_doc.RawDoc() master_doc.merge(migrated_doc) fns = FileNameSource(args.inputs) for filename in fns.generate(): if args.debug: print 'Processing %s' % filename the_file = fmgr.loadFile(filename) lex = lexer.Lexer(dox_tokens.LEXER_TOKENS, skip_whitespace=False) for comment in the_file.comments: # TODO(holtgrew): Also give offset. lex.input(comment.text, filename, comment.line, comment.col, comment.offset_col) parser = dox_parser.Parser() try: parser.parse(lex) except dox_parser.ParserError, e: dox_parser.printParserError(e) return 1 master_doc.merge(parser.documentation)
def TestParsingInfixExpression(): class struct: def __init__(self, input, leftValue, operator, rightValue): self.input = input self.leftValue = leftValue self.operator = operator self.rightValue = rightValue infixTests = [struct("5 + 5;", 5, "+", 5), struct("5 - 5;", 5, "-", 5), struct("5 * 5;", 5, "*", 5), struct("5 / 5;", 5, "/", 5), struct("5 > 5;", 5, ">", 5), struct("5 < 5;", 5, "<", 5), struct("5 != 5;", 5, "!=", 5), struct("true == true", True, "==", True), struct("true != false", True, "!=", False)] for test in infixTests: l = lexer.Lexer(test.input) p = parser.Parser(l) program = p.ParseProgram() checkParserErrors(p) assert len(program.Statements) == 1 stmt = program.Statements[0] assert isinstance(stmt, ast.ExpressionStatement) exp = stmt.Expression TestIntegerLiteral(exp.Left, test.leftValue) assert exp.Operator == test.operator TestIntegerLiteral(exp.Right, test.rightValue) print("TestParsingInfixExpression test is success")
def main(): file_name = __get_file_name(sys.argv) file_ext = __get_file_extension(file_name) if file_ext != ".fyp": eh.print_and_exit("error: invalid file extension: must be .fyp") if not __file_exists(file_name): eh.print_and_exit("error: file does not exist!") _file = open(file_name) _lex = lexer.Lexer(' '.join( [line.rstrip("\n") for line in _file if not __is_comment(line)])) _tokens = _lex.tokenize().tokens is_test = False if len(_tokens) > 1 and _tokens[2] == "test": is_test = True _tokens = _tokens[0:2] + _tokens[3:] _parser = parser_.Parser(_tokens) _eval = evaluator.Evaluator(_parser, is_test) if _eval.tokens[0] != "begin": eh.print_and_exit("error: first function must be 'begin'!") _eval.eval()
def testParse(self): ''' Test the parse() method (parsing an entire tree). ''' class Handler(AbstractHandler): def __init__(self): super(AbstractHandler, self).__init__() self.edge_count = 0 self.node_count = 0 self.tree_end_count = 0 self.leaf_count = 0 def new_edge(self, bootstrap, length): self.edge_count += 1 def new_tree_begin(self): self.node_count += 1 def new_tree_end(self): self.tree_end_count += 1 def new_leaf(self, name): self.leaf_count += 1 l = lexer.Lexer("((A,B),C);") handler = Handler() parser = _Parser(l, handler) parser.parse_node() self.assertEqual(handler.node_count, handler.tree_end_count) self.assertEqual(handler.node_count, 2) self.assertEqual(handler.edge_count, 4) self.assertEqual(handler.leaf_count, 3)
def test_parsing_infix_expressions(): infix_tests = [ ("5 + 5", 5, "+", 5), ("5 - 5", 5, "-", 5), ("5*5", 5, "*", 5), ("5 / 5", 5, "/", 5), ("5 >5", 5, ">", 5), ("5< 5", 5, "<", 5), ("5 == 5", 5, "==", 5), ("5!=5", 5, "!=", 5), ("true == true", True, "==", True), ("true!=false", True, "!=", False), ] for input, left_value, operator, right_value in infix_tests: l = lexer.Lexer(input) p = parser.Parser(l) program = p.parse_program() check_parser_errors(p) check_program_length(program, 1) statement = program.statements[0] check_statement(statement, ast.ExpressionStatement) exp = statement.expression infix_expression_test(exp, left_value, operator, right_value)
def test_operators(self): toks = list(lexer.Lexer("+-*/").make_token()) self.assertEqual(toks, [ tokens.Token(tokens.TokenType.PLUS), tokens.Token(tokens.TokenType.MINUS), tokens.Token(tokens.TokenType.MULTIPLY), tokens.Token(tokens.TokenType.DIVIDE) ])
def test_branch_length_handler(self): ''' Test that the handler works, by calculating the branch total_len sum. ''' l = lexer.Lexer("(('foo' : 0.1, 'bar' : 1.0) : 2, baz)") handler = HandlerTest.BranchLengthSum() p = _Parser(l, handler) p.parse() self.assertEqual(handler.sum, 3.1)
def setUp(self): self.src = source.StringSource() self.lex = lexer.Lexer(self.src) self.environment = env.Environment() self.parser = pars.Parser(self.lex, self.environment) self.program = interpreter.Interpreter(self.environment, self.parser) self.output = io.StringIO() sys.stdout = self.output
def test_numbers(self): toks = list(lexer.Lexer("1245.4 .234 1234. .").make_token()) self.assertEqual(toks, [ tokens.Token(tokens.TokenType.NUMBER, 1245.4), tokens.Token(tokens.TokenType.NUMBER, .234), tokens.Token(tokens.TokenType.NUMBER, 1234.), tokens.Token(tokens.TokenType.NUMBER, 000.000), ])
def main(): content = "" with open('test.lang', 'r') as file: content = file.read() print(content) lex = lexer.Lexer(content) tokens = lex.tokenize()
def get_program(input: str, n_statements: int) -> ast.Program: l = lexer.Lexer(input) p = parser.Parser(l) program = p.parse_program() check_parser_errors(p) check_program_length(program, n_statements) return program