def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(fmt('Token: {0!r} {1!r} {2!s}',
terminals, match, s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max, id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(fmt('Space: {0!r} {1!s}',
terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream,
'No token for {rest} at {location} of {text}.'))
def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(
fmt('Token: {0!r} {1!r} {2!s}', terminals, match,
s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max,
id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(
fmt('Space: {0!r} {1!s}', terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
id_ = s_id(stream)
try:
while not s_empty(stream):
# caches for different tokens with same contents differ
id_ += 1
(line, next_stream) = s_line(stream, False)
line_stream = s_stream(stream, line)
size = 0
# if we use blocks, match leading space
if self.blocks:
try:
(_, size, _) = self.s_regexp.size_match(line_stream)
except TypeError:
pass
# this will be empty (size=0) if blocks unused
(indent, next_line_stream) = s_next(line_stream, count=size)
indent = indent.replace('\t', self._tab)
yield ((START,),
s_stream(line_stream, indent, id_=id_, max=max))
line_stream = next_line_stream
while not s_empty(line_stream):
id_ += 1
try:
(terminals, match, next_line_stream) = \
self.t_regexp.match(line_stream)
yield (terminals, s_stream(line_stream, match,
max=max, id_=id_))
except TypeError:
(terminals, _size, next_line_stream) = \
self.s_regexp.size_match(line_stream)
line_stream = next_line_stream
id_ += 1
yield ((END,),
s_stream(line_stream, '', max=max, id_=id_))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream,
'No token for {rest} at {location} of {text}.'))
def _match(self, in_stream):
'''
Implement matching - pass token stream to tokens.
'''
(max, clean_stream) = s_new_max(in_stream)
try:
length = s_len(in_stream)
except TypeError:
length = None
factory = s_factory(in_stream)
token_stream = factory.to_token(
self._tokens(clean_stream, max),
id=s_id(in_stream), factory=factory,
max=s_max(in_stream),
global_kargs=s_global_kargs(in_stream),
delta=s_delta(in_stream), len=length,
cache_level=s_cache_level(in_stream)+1)
in_stream = None
generator = self.matcher._match(token_stream)
while True:
yield (yield generator)
def _match(self, in_stream):
'''
Implement matching - pass token stream to tokens.
'''
(max, clean_stream) = s_new_max(in_stream)
try:
length = s_len(in_stream)
except TypeError:
length = None
factory = s_factory(in_stream)
token_stream = factory.to_token(self._tokens(clean_stream, max),
id=s_id(in_stream),
factory=factory,
max=s_max(in_stream),
global_kargs=s_global_kargs(in_stream),
delta=s_delta(in_stream),
len=length,
cache_level=s_cache_level(in_stream) +
1)
in_stream = None
generator = self.matcher._match(token_stream)
while True:
yield (yield generator)
def match(support, stream):
# we increment id so that different strings (which might overlap or
# be contiguous) don't affect each other's memoisation (the hash key
# is based on offset and ('one past the') end of one column can have
# the same offset as the start of the next).
id_ = s_id(stream)
# extract a line
(line, next_stream) = s_line(stream, False)
line_stream = s_stream(stream, line)
results = []
for ((left, right), matcher) in zip(indices, matchers):
id_ += 1
# extract the location in the line
(_, left_aligned_line_stream) = s_next(line_stream, count=left)
(word, _) = s_next(left_aligned_line_stream, count=right-left)
support._debug(fmt('Columns {0}-{1} {2!r}', left, right, word))
word_stream = s_stream(left_aligned_line_stream, word, id_=id_)
# do the match
support._debug(s_fmt(word_stream, 'matching {rest}'))
(result, _) = yield matcher._match(word_stream)
results.extend(result)
support._debug(repr(results))
yield (results, next_stream)
def match(support, stream):
# we increment id so that different strings (which might overlap or
# be contiguous) don't affect each other's memoisation (the hash key
# is based on offset and ('one past the') end of one column can have
# the same offset as the start of the next).
id_ = s_id(stream)
# extract a line
(line, next_stream) = s_line(stream, False)
line_stream = s_stream(stream, line)
results = []
for ((left, right), matcher) in zip(indices, matchers):
id_ += 1
# extract the location in the line
(_, left_aligned_line_stream) = s_next(line_stream, count=left)
(word, _) = s_next(left_aligned_line_stream, count=right - left)
support._debug(fmt('Columns {0}-{1} {2!r}', left, right, word))
word_stream = s_stream(left_aligned_line_stream, word, id_=id_)
# do the match
support._debug(s_fmt(word_stream, 'matching {rest}'))
(result, _) = yield matcher._match(word_stream)
results.extend(result)
support._debug(repr(results))
yield (results, next_stream)
