class Lexer():
def __init__(self, stream):
self.stream = Stream(stream)
self.buffer = []
self.tokens = []
self.line = 1
self.col = 1
def read(self):
r = self.stream.read()
if r == "\n":
self.col = 1
return r
def next(self):
r = self.read()
if r != "\x00":
self.buffer.append(r)
return r
def ignore(self):
self.buffer = []
def skip(self):
self.next()
self.ignore()
def fast_forward(self, n):
for i in range(n):
self.next()
def peek(self):
r = self.stream.read()
self.stream.unread()
return r
def emit_value(self, token_type, value):
t = Token((self.line, self.col), token_type, value)
self.tokens.append(t)
print(self.tokens[-1])
self.ignore()
def emit(self, token_type):
self.emit_value(token_type, "".join(self.buffer))
def follow(self, n):
to_unread = 0
for expected in n:
r = self.stream.read()
to_unread += 1
if expected != r:
return False
for i in range(to_unread):
self.stream.unread()
return True
def lex_main(self):
while True:
n = self.peek()
if n == "#":
return self.comment
elif n in ("\n", "\r"):
self.skip()
elif n in (" ", "\t"):
self.skip()
elif n == '"':
return self.string
if n == "\x00":
break
time.sleep(0.25)
return None
def comment(self):
while True:
n = self.peek()
if n == '\n' or n == "\x00":
break
if n == "\r" and self.follow("\r\n"):
break
self.next()
self.ignore()
return self.lex_main
def string(self):
self.skip()
while True:
n = self.peek()
if n == '\n' or n == "\x00":
self.emit(tt.tokenError)
break
if n == "\r" and self.follow("\r\n"):
self.emit(tt.tokenError)
break
if n == '"':
self.emit(tt.tokenString)
self.skip()
break
self.next()
return self.lex_main
def run(self):
state = self.lex_main
while state != None:
state = state()
