def test_init():
parser = new_parser()
parser.init("2 + 3 / 4")
assert isinstance(parser.tokens, list)
expected = [
token(lexer.NUMBER, 2),
token(lexer.PLUS, '+'),
token(lexer.NUMBER, 3),
token(lexer.SLASH, '/'),
token(lexer.NUMBER, 4),
]
eq(len(parser.tokens), len(expected))
for a, b in itertools.izip(parser.tokens, expected):
eq(a, b)
def tokenize(data, lexer):
"""Lex the data and exit.
Args:
data: The data to be lexed.
lexer: The lexer instance to do the lexing.
"""
# Give the lexer some input
lexer.input(data)
# Tokenize
while 1:
tok = lexer.token()
if not tok:
break # No more input
print tok
# Running the lexer twice on the same input screws up the line numbers, so stop here.
sys.exit(0)
# Construimos lexer
lexer = lex.lex(module=lexer)
# Construimos parser
# myparser = myparser()
parser = yacc.yacc(module=myparser, debug=True, debuglog=log)
result = parser.parse(string, lex, debug=log)
datacopy = data
# pasamos la linea como data al lexer
# Esto es con el fin de calcular bien la columna de los tokens
lexer.input(data)
column = 0
controlSpaceSemaphore = True
# Iteramos sobre el la entrada para extraer los tokens
tok = lexer.token()
while tok:
if column == 0:
while column < (tok.lexpos):
output += " "
column += 1
if controlSpaceSemaphore == False:
while column < (tok.lexpos):
output += " "
column += 1
column += len(str(tok.value))
else:
column += len(str(tok.value))
controlSpaceSemaphore = False
if (tok.type == 'TkId' or tok.type == 'TkNum'):
token_info = str(tok.type) + '(valor="' + str(
def peek_next_token(self, n=1):
if self.pos + n < len(self.tokens):
return self.tokens[self.pos+n]
else:
return lexer.token(None, None, 0, 0)
def get_token(self):
if self.pos < len(self.tokens):
return self.tokens[self.pos]
else:
return lexer.token(None, None, 0, 0)
