def test_tokenstream_picklable(): import pickle s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_equals(s.current, Token('a', 1)) s.next() assert_equals(s.current, Token('b', 2)) dumped = pickle.dumps(s) loaded = pickle.loads(dumped) assert_equals(loaded.current, Token('b', 2)) loaded.next() assert_equals(loaded.current, Token('c', 3))
def test_tokenstream(): s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_equals(s.current, Token('a', 1)) # from_tuple_iter s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3)))) assert_equals(s.current, Token('a', 1)) # iter assert_true(isinstance(iter(s), TokenStreamIterator)) assert_equals(tuple(iter(s)), (Token('a', 1), Token('b', 2), Token('c', 3))) # eof s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_false(s.eof) list(s) assert_true(s.eof) # look, push s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_equals(s.current, Token('a', 1)) assert_equals(s.look(), Token('b', 2)) s.next() assert_equals(s.look(), Token('c', 3)) s.push(Token('b', 2)) assert_equals(s.look(), Token('b', 2)) s.push(Token('e', 4), current=True) assert_equals(s.current, Token('e', 4)) assert_equals(s.look(), Token('b', 2)) # skip, next s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) s.skip(1) assert_equals(s.current, Token('b', 2)) s.next() assert_equals(s.current, Token('c', 3)) s.push(Token('e', 4)) assert_equals(s.current, Token('c', 3)) s.next() assert_equals(s.current, Token('e', 4)) s.next() assert_equals(s.current, Token('eof', None)) # expect s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_equals(s.expect('a'), Token('a', 1)) assert_equals(s.expect('b', 2), Token('b', 2)) assert_raises(AssertionError, s.expect, 'e') assert_raises(AssertionError, s.expect, 'c', 5) # test s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) assert_true(s.test('a')) s.next() assert_true(s.test('b', 2)) # shift assert_equals(s.current, Token('b', 2)) s.shift(Token('f', 5)) assert_equals(s.current, Token('f', 5)) s.next() assert_equals(s.current, Token('b', 2))
def tokenize(self, string): """ Resolve quotes and parse quote for quote in an isolated environment. """ buffer = [] stack = [0] open_blocks = [False] def tokenize_buffer(): for item in self.tokenize_block(u'\n'.join(buffer)): yield item del buffer[:] def changes_block_state(line, reverse): primary = self._block_start_re.search secondary = self._block_end_re.search if reverse: primary, secondary = secondary, primary match = primary(line) if match is None: return False while 1: match = secondary(line, match.end()) if match is None: return True match = primary(line, match.end()) if match is None: return False def tokenize_blocks(): for line in string.splitlines(): block_open = open_blocks[-1] if not block_open: m = self._quote_re.match(line) if m is None: level = 0 else: level = len(m.group(1)) line = line[m.end():] if level > stack[-1]: for item in tokenize_buffer(): yield item for new_level in xrange(stack[-1] + 1, level + 1): stack.append(new_level) open_blocks.append(False) yield 'quote_begin', None elif level < stack[-1]: for item in tokenize_buffer(): yield item for x in xrange(stack[-1] - level): stack.pop() open_blocks.pop() yield 'quote_end', None else: line = re.sub('^' + '> ?' * (len(open_blocks) - 1), '', line) if not block_open and changes_block_state(line, False): open_blocks[-1] = True elif block_open and changes_block_state(line, True): open_blocks[-1] = False buffer.append(line) for item in tokenize_buffer(): yield item while stack: if stack.pop(): yield 'quote_end', None open_blocks.pop() return TokenStream.from_tuple_iter(tokenize_blocks())