def open_lex_parse_compare(self, file_path): with open(file_path, 'rb') as f: code = f.read().decode("utf-8") tokens = lex(code, search_match, TokenExpressions) tokens = list( filter(lambda token: token.tokentype_ != TokenTypes.NONE, tokens)) parsed, leftover_token = parse(code, tokens) program = Program(loc_={ 'start': { 'line': 1, 'index': 0 }, "end": { "line": tokens[-1].loc_["start"]["line"], "index": tokens[-1].loc_["start"]["index"] } }, range_=[0, len(code)], body_=parsed) filename, _ = os.path.splitext(os.path.basename(file_path)) folder_output = root_dir + "/tests/parser_tests/test_output/" + os.path.basename( os.path.dirname(file_path)) folder_required = root_dir + "/tests/parser_tests/required/" + os.path.basename( os.path.dirname(file_path)) output_file = folder_output + "/" + filename + ".json" required_file = folder_required + "/" + filename + ".json" if not os.path.exists(folder_output): os.makedirs(folder_output) with open(output_file, "w") as f: f.write(program.jsonify()) self.compare_files(required_file, output_file)
def lex_and_compare_required_vs_output(self, file_path, required_token_order): with open(file_path, 'rb') as f: code = f.read().decode("utf-8") tokens = lex(code, search_match, TokenExpressions) tokens = list( filter(lambda token: token.tokentype_ != TokenTypes.NONE, tokens)) for i in range(len(required_token_order)): self.assertEqual(tokens[i].tokentype_, required_token_order[i], msg=file_path)
if head.tokentype_ in (TokenTypes.EOF, *termination_tokens) : return [], tokens elif head.tokentype_ in (TokenTypes.NEW_LINE, TokenTypes.TAB) : return parse(characters, tail, termination_tokens) elif head.tokentype_ == TokenTypes.VARIABLE_DECLARATION : node, tokens = parse_var_decl.parse_variable_declaration(characters, tokens) elif head.tokentype_ == TokenTypes.FUNCTION_DECLARATION : node, tokens = parse_func_decl.parse_function_declaration(characters, tokens) elif head.tokentype_ == TokenTypes.IF : node, tokens = parse_if_stmt.parse_if_statement(characters, tokens) elif head.tokentype_ == TokenTypes.RETURN : node, tokens = parse_func_decl.parse_return_statement(characters, tokens) elif head.tokentype_ == TokenTypes.CALL : node, tokens = parse_func_call.parse_function_call(characters, tokens) else : return generate_error_message(head, characters, "Invalid Syntax", True) nodes, tokens = parse(characters, tokens, termination_tokens) return [node] + nodes, tokens if __name__ == "__main__": if len(sys.argv) < 2: print("No source file provided") exit() # os.chdir('..') with open(sys.argv[1], 'rb') as f: code = f.read().decode("utf-8") lexed = lex(code, search_match, TokenExpressions) tokens = list(filter(lambda token: token.tokentype_ != TokenTypes.NONE, lexed)) # list(map(print, lexed)) parsed, eof_token = parse(code, tokens) program = Program(loc_={'start': {'line': 1, 'index': 0}, "end":{"line":tokens[-1].loc_["start"]["line"], "index":tokens[-1].loc_["start"]["index"]}}, range_=[0, len(code)], body_=parsed) # print(str(program)) with open("C:\\Users\\Nathan\\Documents\\ATP\\pretty_printed.json", "wb") as f: f.write(program.jsonify().encode("utf-8"))