def test_tokenstream():
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.current, Token('a', 1))
# from_tuple_iter
s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
assert_equals(s.current, Token('a', 1))
# iter
assert_true(isinstance(iter(s), TokenStreamIterator))
assert_equals(tuple(iter(s)),
(Token('a', 1), Token('b', 2), Token('c', 3)))
# eof
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_false(s.eof)
list(s)
assert_true(s.eof)
# look, push
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.current, Token('a', 1))
assert_equals(s.look(), Token('b', 2))
s.next()
assert_equals(s.look(), Token('c', 3))
s.push(Token('b', 2))
assert_equals(s.look(), Token('b', 2))
s.push(Token('e', 4), current=True)
assert_equals(s.current, Token('e', 4))
assert_equals(s.look(), Token('b', 2))
# skip, next
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
s.skip(1)
assert_equals(s.current, Token('b', 2))
s.next()
assert_equals(s.current, Token('c', 3))
s.push(Token('e', 4))
assert_equals(s.current, Token('c', 3))
s.next()
assert_equals(s.current, Token('e', 4))
s.next()
assert_equals(s.current, Token('eof', None))
# expect
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.expect('a'), Token('a', 1))
assert_equals(s.expect('b', 2), Token('b', 2))
assert_raises(AssertionError, s.expect, 'e')
assert_raises(AssertionError, s.expect, 'c', 5)
# test
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_true(s.test('a'))
s.next()
assert_true(s.test('b', 2))
# shift
assert_equals(s.current, Token('b', 2))
s.shift(Token('f', 5))
assert_equals(s.current, Token('f', 5))
s.next()
assert_equals(s.current, Token('b', 2))
def test_tokenstream():
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.current, Token('a', 1))
# from_tuple_iter
s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
assert_equals(s.current, Token('a', 1))
# iter
assert_true(isinstance(iter(s), TokenStreamIterator))
assert_equals(tuple(iter(s)), (Token('a', 1), Token('b', 2), Token('c', 3)))
# eof
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_false(s.eof)
list(s)
assert_true(s.eof)
# look, push
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.current, Token('a', 1))
assert_equals(s.look(), Token('b', 2))
s.next()
assert_equals(s.look(), Token('c', 3))
s.push(Token('b', 2))
assert_equals(s.look(), Token('b', 2))
s.push(Token('e', 4), current=True)
assert_equals(s.current, Token('e', 4))
assert_equals(s.look(), Token('b', 2))
# skip, next
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
s.skip(1)
assert_equals(s.current, Token('b', 2))
s.next()
assert_equals(s.current, Token('c', 3))
s.push(Token('e', 4))
assert_equals(s.current, Token('c', 3))
s.next()
assert_equals(s.current, Token('e', 4))
s.next()
assert_equals(s.current, Token('eof', None))
# expect
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_equals(s.expect('a'), Token('a', 1))
assert_equals(s.expect('b', 2), Token('b', 2))
assert_raises(AssertionError, s.expect, 'e')
assert_raises(AssertionError, s.expect, 'c', 5)
# test
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert_true(s.test('a'))
s.next()
assert_true(s.test('b', 2))
# shift
assert_equals(s.current, Token('b', 2))
s.shift(Token('f', 5))
assert_equals(s.current, Token('f', 5))
s.next()
assert_equals(s.current, Token('b', 2))
