def assert_tokens(source, expected, get_tokens=get_tokens, data_only=False):
tokens = list(get_tokens(source, data_only))
assert_equal(len(tokens),
len(expected),
'Expected %d tokens:\n%s\n\nGot %d tokens:\n%s' %
(len(expected), expected, len(tokens), tokens),
values=False)
for act, exp in zip(tokens, expected):
assert_equal(act, Token(*exp), formatter=repr)
def assert_tokens(source, expected, get_tokens=get_tokens, data_only=False):
tokens = list(get_tokens(source, data_only))
assert_equal(len(tokens),
len(expected),
'Expected %d tokens:\n%s\n\nGot %d tokens:\n%s' %
(len(expected), expected, len(tokens), tokens),
values=False)
for act, exp in zip(tokens, expected):
exp = Token(*exp)
assert_equal(act.type, exp.type)
assert_equal(act.value, exp.value, formatter=repr)
assert_equal(act.lineno, exp.lineno)
assert_equal(act.columnno, exp.columnno)
def _verify(self, source, data_only=False):
tokens = get_tokens(source, data_only)
expected = self.data_tokens if data_only else self.tokens
assert_tokens(list(tokens), expected)