def test_tokenizes_single_line_comment(self):
self.check("// asd", [COMMENT(value=" asd")])
self.check("asd// asd",
[IDENTIFIER(value="asd"),
COMMENT(value=" asd")])
self.check("asd// asd //",
[IDENTIFIER(value="asd"),
COMMENT(value=" asd //")])
def test_tokenizes_define(self):
self.check(
"`define name",
[
PREPROCESSOR(value="define"),
WHITESPACE(value=" "),
IDENTIFIER(value="name"),
],
)
def test_has_location_information(self):
self.check(
"`define foo",
[
PREPROCESSOR(value="define", location=(("fn.v",
(0, 6)), None)),
WHITESPACE(value=" ", location=(("fn.v", (7, 7)), None)),
IDENTIFIER(value="foo", location=(("fn.v", (8, 10)), None)),
],
strip_loc=False,
)
def test_escaped_newline_ignored(self):
self.check("a\\\nb", [IDENTIFIER(value="a"), IDENTIFIER(value="b")])
def test_tokenizes_newline(self):
self.check("asd\n", [IDENTIFIER(value="asd"), NEWLINE(value="")])
def test_tokenizes_semi_colon(self):
self.check("asd;", [IDENTIFIER(value="asd"), SEMI_COLON(value="")])