def _get_token_list(string, version=None):
# Load the current version.
version_info = parse_version_string(version)
return list(tokenize(string, version_info))
def __init__(self, bnf_grammar: str) -> None:
self._bnf_grammar: str = bnf_grammar
self.generator = tokenize(bnf_grammar, version_info=parse_version_string("3.6"))
self._gettoken() # Initialize lookahead
def test_tokenize_start_pos(self, code, positions):
tokens = list(tokenize(code, version_info=parse_version_string("3.6")))
assert positions == [p.start_pos for p in tokens]