def tokenizer(handle, interactive=False):
"""Takes a file-like object and produces a stream of tokens following
the LISP rules.
If interative is True, the file reading proceeds char-by-char with
no buffering. This is useful for interactive use for example with
a SMT-Lib2-compliant solver
"""
spaces = set([" ", "\n", "\t"])
separators = set(["(", ")", "|"])
specials = spaces | separators | set([";", ""])
if not interactive:
reader = itertools.chain.from_iterable(handle) # reads char-by-char
else:
reader = interactive_char_iterator(handle)
c = next(reader)
eof = False
while not eof:
if c in specials:
# consume the spaces
if c in spaces:
c = next(reader)
elif c in separators:
if c == "|":
s = []
c = next(reader)
while c and c != "|":
if c == "\\": # This is a single '\'
c = next(reader)
if c != "|" and c != "\\":
# Only \| and \\ are supported escapings
raise SyntaxError("Unknown escaping in " \
"quoted symbol: '\\%s'" % c)
s.append(c)
c = next(reader)
if not c:
raise SyntaxError("Expected '|'")
yield ("".join(s))
else:
yield c
c = next(reader)
elif c == ";":
while c and c != "\n":
c = next(reader)
c = next(reader)
else:
# EOF
eof = True
assert len(c) == 0
else:
tk = []
while c not in specials:
tk.append(c)
c = next(reader)
yield "".join(tk)
def tokenizer(handle, interactive=False):
"""Takes a file-like object and produces a stream of tokens following
the LISP rules.
If interative is True, the file reading proceeds char-by-char with
no buffering. This is useful for interactive use for example with
a SMT-Lib2-compliant solver
"""
spaces = set([" ", "\n", "\t"])
separators = set(["(", ")", "|"])
specials = spaces | separators | set([";", ""])
if not interactive:
reader = itertools.chain.from_iterable(handle) # reads char-by-char
else:
reader = interactive_char_iterator(handle)
c = next(reader)
eof = False
while not eof:
if c in specials:
# consume the spaces
if c in spaces:
c = next(reader)
elif c in separators:
if c == "|":
s = []
c = next(reader)
while c and c != "|":
if c == "\\": # This is a single '\'
c = next(reader)
if c != "|" and c != "\\":
# Only \| and \\ are supported escapings
raise SyntaxError("Unknown escaping in " \
"quoted symbol: '\\%s'" % c)
s.append(c)
c = next(reader)
if not c:
raise SyntaxError("Expected '|'")
yield ("".join(s))
else:
yield c
c = next(reader)
elif c == ";":
while c and c != "\n":
c = next(reader)
c = next(reader)
else:
# EOF
eof = True
assert len(c) == 0
else:
tk = []
while c not in specials:
tk.append(c)
c = next(reader)
yield "".join(tk)
def __init__(self, handle, interactive=False):
if not interactive:
# reads char-by-char
self.reader = itertools.chain.from_iterable(handle)
else:
self.reader = interactive_char_iterator(handle)
self.generator = self.create_generator(self.reader)
self.extra_queue = []
self.consume = self.consume_token
def __init__(self, handle, interactive=False):
if not interactive:
# reads char-by-char
self.reader = itertools.chain.from_iterable(handle)
else:
self.reader = interactive_char_iterator(handle)
self.generator = self.create_generator(self.reader)
self.extra_queue = []
self.consume = self.consume_token
