def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(
fmt('Token: {0!r} {1!r} {2!s}', terminals, match,
s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max,
id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(
fmt('Space: {0!r} {1!s}', terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(fmt('Token: {0!r} {1!r} {2!s}',
terminals, match, s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max, id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(fmt('Space: {0!r} {1!s}',
terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream,
'No token for {rest} at {location} of {text}.'))
def __repr__(self):
'''
Lazily evaluated for speed - saves 1/3 of time spent in constructor
'''
if not self.__cached_repr:
try:
s = s_debug(self.stream)
except AttributeError:
s = '<stream>'
self.__cached_repr = fmt('{0}({1})', self.matcher, s)
return self.__cached_repr
def __repr__(self):
'''
Lazily evaluated for speed - saves 1/3 of time spent in constructor
'''
if not self.__cached_repr:
try:
s = s_debug(self.stream)
except AttributeError:
s = '<stream>'
self.__cached_repr = fmt('{0}({1})', self.matcher, s)
return self.__cached_repr
def test_all(self):
lines = iter(['first line', 'second line', 'third line'])
f = DEFAULT_STREAM_FACTORY
s1 = f(lines)
# just created
assert not s_empty(s1)
# get first line
(l1, s2) = s_line(s1, False)
assert 'first line' == l1, l1
# get first character of next line
(c21, s21) = s_next(s2)
assert c21 == 's', c21
# and test fmtting
locn = s_fmt(s21, '{location}: {rest}')
assert locn == "line 2, character 2: 'econd line'", locn
# then get rest of second line
(c22, s3) = s_next(s21, count=len('econd line'))
assert c22 == 'econd line', c22
d = s_debug(s21)
assert d == "1:'e'", d
# and move on to third line
(c31, s31) = s_next(s3)
assert c31 == 't', c31
(c32, s32) = s_next(s31)
assert c32 == 'h', c32
# now try branching (think tokens) at line 1
s10 = s_stream(s2, l1)
(l1, s20) = s_line(s10, False)
assert l1 == 'first line', l1
assert not s_empty(s20)
(c1, s11) = s_next(s10)
assert c1 == 'f', c1
d = s_debug(s11)
assert d == "1:'i'", d
# finally look at max depth (which was after 'h' in third line)
m = s_deepest(s1)
locn = s_fmt(m, '{location}: {rest}')
assert locn == "line 3, character 3: 'ird line'", locn
def debug(self, state):
try:
line_stream = state_to_line_stream(state)
return s_debug(line_stream)
except StopIteration:
return '<EOS>'
def debug(self, state):
try:
line_stream = state_to_line_stream(state)
return s_debug(line_stream)
except StopIteration:
return '<EOS>'
