def test_tokenize_binary_operator(): """Ensure that binary operators don't have surrounding whitespace. Regression test. """ code = "foo == bar" assert [i.value for i in tokenize(code)] == ["foo", "==", "bar"]
def test_no_matching_token(): """Ensure that we handle cases when there is no match.""" code = """ foo """ assert not list(match_tokens(tokenize(code), start=match_regex("foo"), end=match_regex("bar")))
def test_no_matching_token(): """Ensure that we handle cases when there is no match.""" code = """ foo """ assert not list( match_tokens( tokenize(code), start=match_regex("foo"), end=match_regex("bar")))
def test_not_enough_lookahead(): """Ensure that we don't return a match if there's not enough lookahead.""" code = """ foo bar """ assert not list(match_tokens(tokenize(code), start=match_regex("foo"), end=match_regex("bar"), lookahead=1))
def test_tokenize_single_line_comment(): """Ensure that we tokenize single-line comments correctly.""" assert [i.value for i in tokenize(r""" struct foo; // this is a comment with bad code: struct foo; """.strip())] == [ "struct", "foo", ";", "// this is a comment with bad code: struct foo;", ]
def test_not_enough_lookahead(): """Ensure that we don't return a match if there's not enough lookahead.""" code = """ foo bar """ assert not list( match_tokens(tokenize(code), start=match_regex("foo"), end=match_regex("bar"), lookahead=1))
def test_tokenize_backslash_string_literal(): """Ensure that we handle backslash-escapes correctly in strings.""" code = r""" "foo\"bar\\baz" """.strip() assert tokenize(code) == [ Token(type="string", value=code, start=Position(row=0, column=0), end=Position(row=0, column=14)), ]
def test_with_matching_tokens(): """Ensure that we can use the decorator to match tokens.""" code = """ foo bar """ @with_matched_tokens(start=match_regex("foo"), end=match_regex("bar")) def func(tokens, match): yield "baz" assert list(func(tokenize(code))) == ["baz"]
def test_match_length(): """Ensure that we correctly find sequences of a provided length.""" code = """ foo foo bar """ result = match_tokens(tokenize(code), start=match_regex("foo"), end=match_regex("bar"), length=3) result = list(result) assert len(result) == 1 assert [i.value for i in result[0]] == ["foo", "foo", "bar"]
def test_with_matching_tokens(): """Ensure that we can use the decorator to match tokens.""" code = """ foo bar """ @with_matched_tokens(start=match_regex("foo"), end=match_regex("bar")) def func(source, *, match): yield "baz" source_code = SourceCode(filename="foo.cpp", tokens=tokenize(code)) assert list(func(source_code)) == ["baz"]
def test_tokenize(): """Ensure that we tokenize code correctly.""" assert tokenize(""" int main() { } foo (); """.strip()) == [ Token(type="keyword", start=Position(row=0, column=0), end=Position(row=0, column=2), value="int"), Token(type="identifier", start=Position(row=0, column=4), end=Position(row=0, column=7), value="main"), Token(type="grouping", start=Position(row=0, column=8), end=Position(row=0, column=8), value="("), Token(type="grouping", start=Position(row=0, column=9), end=Position(row=0, column=9), value=")"), Token(type="grouping", start=Position(row=0, column=11), end=Position(row=0, column=11), value="{"), Token(type="grouping", start=Position(row=2, column=0), end=Position(row=2, column=0), value="}"), Token(type="identifier", start=Position(row=3, column=0), end=Position(row=3, column=2), value="foo"), Token(type="grouping", start=Position(row=3, column=4), end=Position(row=3, column=4), value="("), Token(type="grouping", start=Position(row=3, column=5), end=Position(row=3, column=5), value=")"), Token(type="grouping", start=Position(row=3, column=6), end=Position(row=3, column=6), value=";"), ]
def test_match_tokens(): """Ensure that we match tokens correctly.""" code = """ foo qux bar foo grault bar foo bar """ expected = [["foo", "qux", "bar"], ["foo", "grault", "bar"], ["foo", "bar"]] assert expected == [[ token.value for token in match ] for match in match_tokens( tokenize(code), start=match_regex("foo"), end=match_regex("bar"))]
def test_tokenize_multiline_comment(): """Ensure that we tokenize multiline comments.""" assert tokenize(r""" foo /* bar baz */ """.strip()) == [ Token(type="identifier", value="foo", start=Position(row=0, column=0), end=Position(row=0, column=2)), Token(type="comment", value="/* bar\nbaz */", start=Position(row=1, column=0), end=Position(row=2, column=5)), ]
def test_match_tokens(): """Ensure that we match tokens correctly.""" code = """ foo qux bar foo grault bar foo bar """ expected = [["foo", "qux", "bar"], ["foo", "grault", "bar"], ["foo", "bar"]] assert expected == [ [token.value for token in match] for match in match_tokens(tokenize(code), start=match_regex("foo"), end=match_regex("bar"))]
def test_tokenize_string(): """Ensure that we tokenize strings correctly.""" assert tokenize(""" hello = "foo" ; """.strip()) == [ Token(type="identifier", value="hello", start=Position(row=0, column=0), end=Position(row=0, column=4)), Token(type="binary_operator", value="=", start=Position(row=0, column=6), end=Position(row=0, column=6)), Token(type="string", value='"foo"', start=Position(row=0, column=8), end=Position(row=0, column=12)), Token(type="grouping", value=";", start=Position(row=1, column=0), end=Position(row=1, column=0)), ]
def test_tokenize_unterminated_multiline_comment(): """Ensure that we reject unterminated multiline comments.""" with pytest.raises(ValueError): tokenize("/*")
def test_tokenize_unterminated_string_literal(): """Ensure that we reject unterminated string literals.""" with pytest.raises(ValueError): tokenize(r""" "foo """)
def test_fail_to_parse_token(): """Ensure that we reject code with unknown tokens.""" with pytest.raises(ValueError): tokenize(r"`")