def _tokens(self, stream, max): ''' Generate tokens, on demand. ''' try: id_ = s_id(stream) while not s_empty(stream): # avoid conflicts between tokens id_ += 1 try: (terminals, match, next_stream) = \ self.t_regexp.match(stream) self._debug(fmt('Token: {0!r} {1!r} {2!s}', terminals, match, s_debug(stream))) yield (terminals, s_stream(stream, match, max=max, id_=id_)) except TypeError: (terminals, _size, next_stream) = \ self.s_regexp.size_match(stream) self._debug(fmt('Space: {0!r} {1!s}', terminals, s_debug(stream))) stream = next_stream except TypeError: raise RuntimeLexerError( s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def _tokens(self, stream, max): ''' Generate tokens, on demand. ''' try: id_ = s_id(stream) while not s_empty(stream): # avoid conflicts between tokens id_ += 1 try: (terminals, match, next_stream) = \ self.t_regexp.match(stream) self._debug( fmt('Token: {0!r} {1!r} {2!s}', terminals, match, s_debug(stream))) yield (terminals, s_stream(stream, match, max=max, id_=id_)) except TypeError: (terminals, _size, next_stream) = \ self.s_regexp.size_match(stream) self._debug( fmt('Space: {0!r} {1!s}', terminals, s_debug(stream))) stream = next_stream except TypeError: raise RuntimeLexerError( s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def _tokens(self, stream, max): ''' Generate tokens, on demand. ''' id_ = s_id(stream) try: while not s_empty(stream): # caches for different tokens with same contents differ id_ += 1 (line, next_stream) = s_line(stream, False) line_stream = s_stream(stream, line) size = 0 # if we use blocks, match leading space if self.blocks: try: (_, size, _) = self.s_regexp.size_match(line_stream) except TypeError: pass # this will be empty (size=0) if blocks unused (indent, next_line_stream) = s_next(line_stream, count=size) indent = indent.replace('\t', self._tab) yield ((START,), s_stream(line_stream, indent, id_=id_, max=max)) line_stream = next_line_stream while not s_empty(line_stream): id_ += 1 try: (terminals, match, next_line_stream) = \ self.t_regexp.match(line_stream) yield (terminals, s_stream(line_stream, match, max=max, id_=id_)) except TypeError: (terminals, _size, next_line_stream) = \ self.s_regexp.size_match(line_stream) line_stream = next_line_stream id_ += 1 yield ((END,), s_stream(line_stream, '', max=max, id_=id_)) stream = next_stream except TypeError: raise RuntimeLexerError( s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def _match(self, in_stream): ''' Implement matching - pass token stream to tokens. ''' (max, clean_stream) = s_new_max(in_stream) try: length = s_len(in_stream) except TypeError: length = None factory = s_factory(in_stream) token_stream = factory.to_token( self._tokens(clean_stream, max), id=s_id(in_stream), factory=factory, max=s_max(in_stream), global_kargs=s_global_kargs(in_stream), delta=s_delta(in_stream), len=length, cache_level=s_cache_level(in_stream)+1) in_stream = None generator = self.matcher._match(token_stream) while True: yield (yield generator)
def _match(self, in_stream): ''' Implement matching - pass token stream to tokens. ''' (max, clean_stream) = s_new_max(in_stream) try: length = s_len(in_stream) except TypeError: length = None factory = s_factory(in_stream) token_stream = factory.to_token(self._tokens(clean_stream, max), id=s_id(in_stream), factory=factory, max=s_max(in_stream), global_kargs=s_global_kargs(in_stream), delta=s_delta(in_stream), len=length, cache_level=s_cache_level(in_stream) + 1) in_stream = None generator = self.matcher._match(token_stream) while True: yield (yield generator)
def match(support, stream): # we increment id so that different strings (which might overlap or # be contiguous) don't affect each other's memoisation (the hash key # is based on offset and ('one past the') end of one column can have # the same offset as the start of the next). id_ = s_id(stream) # extract a line (line, next_stream) = s_line(stream, False) line_stream = s_stream(stream, line) results = [] for ((left, right), matcher) in zip(indices, matchers): id_ += 1 # extract the location in the line (_, left_aligned_line_stream) = s_next(line_stream, count=left) (word, _) = s_next(left_aligned_line_stream, count=right-left) support._debug(fmt('Columns {0}-{1} {2!r}', left, right, word)) word_stream = s_stream(left_aligned_line_stream, word, id_=id_) # do the match support._debug(s_fmt(word_stream, 'matching {rest}')) (result, _) = yield matcher._match(word_stream) results.extend(result) support._debug(repr(results)) yield (results, next_stream)
def match(support, stream): # we increment id so that different strings (which might overlap or # be contiguous) don't affect each other's memoisation (the hash key # is based on offset and ('one past the') end of one column can have # the same offset as the start of the next). id_ = s_id(stream) # extract a line (line, next_stream) = s_line(stream, False) line_stream = s_stream(stream, line) results = [] for ((left, right), matcher) in zip(indices, matchers): id_ += 1 # extract the location in the line (_, left_aligned_line_stream) = s_next(line_stream, count=left) (word, _) = s_next(left_aligned_line_stream, count=right - left) support._debug(fmt('Columns {0}-{1} {2!r}', left, right, word)) word_stream = s_stream(left_aligned_line_stream, word, id_=id_) # do the match support._debug(s_fmt(word_stream, 'matching {rest}')) (result, _) = yield matcher._match(word_stream) results.extend(result) support._debug(repr(results)) yield (results, next_stream)