def test_init(): parser = new_parser() parser.init("2 + 3 / 4") assert isinstance(parser.tokens, list) expected = [ token(lexer.NUMBER, 2), token(lexer.PLUS, '+'), token(lexer.NUMBER, 3), token(lexer.SLASH, '/'), token(lexer.NUMBER, 4), ] eq(len(parser.tokens), len(expected)) for a, b in itertools.izip(parser.tokens, expected): eq(a, b)
def tokenize(data, lexer): """Lex the data and exit. Args: data: The data to be lexed. lexer: The lexer instance to do the lexing. """ # Give the lexer some input lexer.input(data) # Tokenize while 1: tok = lexer.token() if not tok: break # No more input print tok # Running the lexer twice on the same input screws up the line numbers, so stop here. sys.exit(0)
# Construimos lexer lexer = lex.lex(module=lexer) # Construimos parser # myparser = myparser() parser = yacc.yacc(module=myparser, debug=True, debuglog=log) result = parser.parse(string, lex, debug=log) datacopy = data # pasamos la linea como data al lexer # Esto es con el fin de calcular bien la columna de los tokens lexer.input(data) column = 0 controlSpaceSemaphore = True # Iteramos sobre el la entrada para extraer los tokens tok = lexer.token() while tok: if column == 0: while column < (tok.lexpos): output += " " column += 1 if controlSpaceSemaphore == False: while column < (tok.lexpos): output += " " column += 1 column += len(str(tok.value)) else: column += len(str(tok.value)) controlSpaceSemaphore = False if (tok.type == 'TkId' or tok.type == 'TkNum'): token_info = str(tok.type) + '(valor="' + str(
def peek_next_token(self, n=1): if self.pos + n < len(self.tokens): return self.tokens[self.pos+n] else: return lexer.token(None, None, 0, 0)
def get_token(self): if self.pos < len(self.tokens): return self.tokens[self.pos] else: return lexer.token(None, None, 0, 0)