def test_invalid_tokenizing(): for token_type in invalid_tokens: for source, expected_match in invalid_tokens[token_type]: token = _munch_a_token(source) if token: assert not (token.type == token_type and token.source_substring == expected_match)
def test_valid_tokenizing(): for token_type in valid_tokens: for (source, expected_match) in valid_tokens[token_type]: token = _munch_a_token(source) assert token, "Failed to tokenize: {}\nExpected: {}\nOut of: {}\nGot nothing!".format( token_type, expected_match, source) assert token.type == token_type, \ "Expected type: {}\nOut of: {}\nThat matched: {}\nOf type: {}".format( token_type, source, token.source_substring, token.type) assert token.source_substring == expected_match