def include_end(self): ''' Performs and end of include. ''' self.lex = self.filestack[-1][2] self.input_data = self.filestack[-1][3] self.filestack.pop() if self.filestack == []: # End of input? return self.filestack[-1][1] += 1 # Increment line counter of previous file result = lex.LexToken() # Creates the token result.value = self.put_current_line() result.type = '_ENDFILE_' result.lineno = self.lex.lineno result.lexpos = self.lex.lexpos return result
def include_end(self): """ Performs and end of include. """ self.lex = self.filestack[-1][2] self.input_data = self.filestack[-1][3] self.filestack.pop() if not self.filestack: # End of input? return self.filestack[-1][1] += 1 # Increment line counter of previous file result = lex.LexToken() result.value = self.put_current_line(suffix='\n') result.type = '_ENDFILE_' result.lineno = self.lex.lineno result.lexpos = self.lex.lexpos return result
def trace(lexer, *argv): token = lexer.token_() if not quiet_mode: sys.stderr.write("TOKEN ") sys.stderr.write(repr( token ) ) sys.stderr.write("\n") sys.stderr.flush() if token is not None: return token if len(lexer.indentation_stack) == 1: return lexer.indentation_stack.pop() token = lex.LexToken() token.value = '' token.type = 'DEDENT' token.lineno = lexer.lineno token.lexpos = lexer.lexpos token.lexer = lexer return token
def t_INCLUDE(t): r'\#[ \t]*include[ \t]+\"(.|(\\[ntr0\'"])|(\\x[0-9a-fA-F][0-9a-fA-F]))*?\"' include_file = re.split('[ \t]+', t.value)[-1].strip('\"') global last_newline if t.lexpos != last_newline: print('Error: "#include" is not at the beginning of the line!') pass elif include_file in included.keys(): print('Warning: File "%s" already included!' % include_file) pass elif not os.path.isfile(include_file): print('Error: No such file or directory: "%s"!' % include_file) sys.exit() elif not os.access(include_file, os.R_OK): print('Error: Can\'t open file or directory: "%s"!' % include_file) sys.exit() else: token_list.append(t) included[include_file] = '1' include_lexer = lex.lex() include_fin = open(include_file, 'r') include_data = include_fin.read() include_fin.close() last_newline = 0 include_lexer.input(include_data) while True: toke = include_lexer.token() if not toke: break # No more input token_list.append(toke) t.value = include_file kalimera = lex.LexToken() kalimera.type = 'END_INCLUDE' kalimera.value = t.value kalimera.lineno = 0 kalimera.lexpos = 0 token_list.append(kalimera) pass
def create_py_blocks(self, token_stream): for tok in token_stream: if not tok.type == 'PY_BLOCK_START': yield tok continue # yield the start token since it's needed by the parser start_tok = tok yield start_tok # The next token must be a newline or its a syntax error try: nl_tok = token_stream.next() except StopIteration: nl_tok = None if nl_tok is None or nl_tok.type != 'NEWLINE': if nl_tok is None: # create a fake token with a line number # for the error handler. nl_tok = lex.LexToken() nl_tok.lineno = start_tok.lineno msg = 'Newline required after a ":: python ::" tag' raise_syntax_error(msg, nl_tok) # yield the newline token since it's needed by the parser yield nl_tok # Collect the Python code from the block py_toks = [] end_tok = None for tok in token_stream: if tok.type == 'PY_BLOCK_END': end_tok = tok break elif tok.type == 'NEWLINE': py_toks.append(tok) else: assert tok.type == 'PY_BLOCK_CONTINUE', tok.type py_toks.append(tok) if end_tok is None: # Reach end of input without an :: end :: delimiter msg = 'EOF while scanning raw python block' raise_syntax_error(msg, start_tok) # Create the python text to add to the py block token # creating blank lines as necessary so that syntax errors # get reported with correct line numbers. The captured # text gets handed directly to Python's compile function. leader = '\n' * start_tok.lineno py_txt = leader + ''.join(tok.value for tok in py_toks) # create a python token py_block = lex.LexToken() py_block.lineno = start_tok.lineno + 1 py_block.lexpos = -1 py_block.value = py_txt py_block.type = 'PY_BLOCK' # Yield the py block to the parser yield py_block # Yield the end block to the parser yield end_tok # An end token must be followed by a newline try: nl_tok = token_stream.next() except StopIteration: nl_tok = None if nl_tok is None or nl_tok.type != 'NEWLINE': if nl_tok is None: # create a fake token with a line number # for the error handler. nl_tok = lex.LexToken() nl_tok.lineno = end_tok.lineno msg = 'Newline required after a ":: end ::" tag' raise_syntax_error(msg, nl_tok) # The parser requires the newline token yield nl_tok
def makeToken(value, type): """Helper function to quickly make a token""" tok = lex.LexToken() tok.value = value tok.type = type return tok
def p_def_func(p): 'def_func : id opar arg_list cpar colon type ocur expr ccur semi' p[0] = FuncDeclarationNode(p[1], p[3], p[6], p[8]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def _new_token(type, lineno): tok = lex.LexToken() tok.type = type tok.value = None tok.lineno = lineno return tok
def p_program(p): 'program : class_list' p[0] = ProgramNode(p[1]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def t_comment(dummy_t): r'\#.*' pass def t_error(t): ''' Prints error messages when scan fails ''' print("Illegal character at line {} '{}'".format(t.lexer.lineno, \ t.value[0])) t.lexer.skip(1) import ply.lex as lex lexer = lex.lex(debug=0) # Build the lexer EOF = lex.LexToken() EOF.type = 'EOF' ############################################################################### # Classes that define nodes in a tree of nodes constructed by the parser ############################################################################### class NumNode: def __init__(self, num): self.num = num def run(self): return self.num def generate(self): return str(self.num)
def p_expr_while(p): 'expr : while expr loop expr pool' p[0] = WhileNode(p[2], p[4]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_instantiate(p): 'expr : new type' p[0] = InstantiateNode(p[2]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_isvoid(p): 'expr : isvoid expr' p[0] = IsVoidNode(p[2]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_case(p): 'expr : case expr of case_list esac' p[0] = SwitchCaseNode(p[2], p[4]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_let(p): 'expr : let decl_list in expr' p[0] = LetInNode(p[2], p[4]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_chunk(p): 'expr : ocur chunk ccur' p[0] = ChunkNode(p[2]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_assign(p): 'expr : id assignArrow expr' p[0] = AssignNode(p[1], p[3]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def get_dedent(self): return lex.LexToken('DEDENT', 1, self.lexer.lineno) # important
def p_expr_id(p): 'expr : id' p[0] = VariableNode(p[1]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def p_expr_if(p): 'expr : if expr then expr else expr fi' p[0] = ConditionalNode(p[2], p[4], p[6]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def get_newline(self): return lex.LexToken('NEWLINE', 1, self.lexer.lineno)
def p_expr_int(p): 'expr : number' p[0] = ConstantNumNode(p[1]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]
def _MakeLexToken(token_type, value, lineno=1, lexpos=0): """Makes a LexToken with the given parameters. (Note that lineno is 1-based, but lexpos is 0-based.)""" rv = lex.LexToken() rv.type, rv.value, rv.lineno, rv.lexpos = token_type, value, lineno, lexpos return rv
def p_expr_string(p): 'expr : string' p[0] = StringNode(p[1]) p[0].token_list = [ sl for sl in p.slice if type(lt.LexToken()) == type(sl) ]