def check_lex(self, tokens, spec):
"""Check that the provided spec parses to the provided token list."""
lex_output = sp.SpecLexer().lex(spec)
for tok, spec_tok in zip(tokens, lex_output):
if tok.type == sp.ID:
self.assertEqual(tok, spec_tok)
else:
# Only check the type for non-identifiers.
self.assertEqual(tok.type, spec_tok.type)
def check_lex(self, tokens, spec):
"""Check that the provided spec parses to the provided token list."""
spec = shlex.split(spec)
lex_output = sp.SpecLexer().lex(spec)
for tok, spec_tok in zip(tokens, lex_output):
if tok.type == sp.ID or tok.type == sp.VAL:
assert tok == spec_tok
else:
# Only check the type for non-identifiers.
assert tok.type == spec_tok.type