def test_trigraphs(self):
src = "??( ??) ??/ ??' ??< ??> ??! ??- ??="
source_file = SourceFile('a.h')
chars = list(lexer.create_characters(io.StringIO(src), source_file))
chars = list(lexer.trigraph_filter(chars))
self.assertSequenceEqual(list(r'[ ] \ ^ { } | ~ #'),
[c.char for c in chars])
self.assertSequenceEqual([1] * 17, [c.loc.row for c in chars])
self.assertSequenceEqual(
[1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29, 32, 33],
[c.loc.col for c in chars])
def test_generate_characters(self):
src = "ab\ndf"
source_file = SourceFile("a.h")
chars = list(lexer.create_characters(io.StringIO(src), source_file))
self.assertSequenceEqual([1, 1, 1, 2, 2], [c.loc.row for c in chars])
self.assertSequenceEqual([1, 2, 3, 1, 2], [c.loc.col for c in chars])
def tokenize(self, src):
source_file = SourceFile("a.h")
tokens = list(self.lexer.lex(io.StringIO(src), source_file))
return tokens