def test_string_multiline_slash(self):
t = Tokenizer(PythonSchema)
res = list(t.tokenize("'foo\\"))
assert res == [Token("'foo\\", type='string')]
res = list(t.tokenize("bar'"))
assert res == [Token("bar'", type='string')]
res = list(t.tokenize("bar"))
assert res == [Token('bar', type='word')]
res = list(t.tokenize('"foo\\bar"'))
assert res == [Token('"foo\\bar"', type="string")]
def test_string_multiline(self):
t = Tokenizer(PythonSchema)
res = list(t.tokenize('"""foo\n'))
assert res == [Token('"""foo\n', type='string')]
res = list(t.tokenize('bar\n'))
assert res == [Token('bar\n', type='string')]
res = list(t.tokenize('"""\n'))
assert res == [Token('"""', type='string'),
Token('\n', type='whitespace')]
# tricky problem: the following line must not put the tokenizer in
# 'multiline state'...
res = list(t.tokenize('"""foo"""'))
assert res == [Token('"""foo"""', type='string')]
res = list(t.tokenize('bar'))
assert res == [Token('bar', type='word')]
def test_something_strange(self):
t = Tokenizer(PythonSchema)
tokens = list(t.tokenize('"""foo "bar" baz"""'))
assert not t._inside_multiline
def tokens(self, data):
t = Tokenizer(PythonSchema)
return list(t.tokenize(data))
