def test_token_stream_iterator(): #Low level tests, for more coverage stream = TokenStream.from_tuple_iter(TEST_STREAM_TUPLE) assert_true(isinstance(iter(stream), TokenStreamIterator)) # check that TokenStreamIterator.__iter__ works as expected (required for coverage) assert_true(isinstance(iter(iter(stream)), TokenStreamIterator)) iter_ = iter(stream) assert_equal(iter_._stream.current.type, 'bold') iter_.next() assert_equal(iter_._stream.current.type, 'italic')
def test_look(): stream = TokenStream.from_tuple_iter(TEST_STREAM) for iexp, exp in enumerate(TEST_STREAM): new = stream.look() if new.type != 'eof': assert_equal(TEST_STREAM[iexp+1].as_tuple(), new.as_tuple()) stream.next() # this is a bit fancy, but imho the right behaviour # XXX: does this belong here and not to `test_feed`? stream.push(Token('fooobaaaar')) assert_equal(stream.current.type, 'eof') assert_equal(stream.look().type, 'fooobaaaar') # skip the current 'eof' token and the 'fooobaaaar' token stream.skip(2) assert_equal(stream.current.type, 'eof')
def tokenize(self, raw=None, enable_escaping=False): """ Tokenize the raw document, apply stream-filters and return the processing-ready token stream. :param raw: The raw document. :return: A `TokenStream` instance. """ ctx = Context(self, enable_escaping) stream = TokenStream.from_tuple_iter(self._process_lexing_rules(raw or self.raw, enable_escaping)) for callback in events.iter_callbacks("process-stream"): ret = callback(stream, ctx) if ret is not None: stream = ret return stream
def tokenize(self, raw=None, enable_escaping=False): """ Tokenize the raw document, apply stream-filters and return the processing-ready token stream. :param raw: The raw document. :return: A `TokenStream` instance. """ ctx = Context(self, enable_escaping) stream = TokenStream.from_tuple_iter( self._process_lexing_rules(raw or self.raw, enable_escaping)) for callback in events.iter_callbacks('process-stream'): ret = callback(stream, ctx) if ret is not None: stream = ret return stream