def validate_get_tokens_reentrant(*buffers):
tokens = []
buffer = Buffer('')
for b in buffers:
buffer = buffer + b
tokens += list(get_tokens(buffer))
tokens += list(get_tokens(buffer, False))
unfinished = buffer.search()
assert not unfinished
assert len(tokens) == len(RAW_TOKENS)
assert tokens == RAW_TOKENS
def test_get_tokens_all():
buf = Buffer(RAW_DATA)
parser = get_tokens(buf, more_data=False)
tokens = list(parser)
assert len(tokens) == len(RAW_TOKENS)
assert tokens == RAW_TOKENS
