def test_string_multiline_slash(self): t = Tokenizer(PythonSchema) res = list(t.tokenize("'foo\\")) assert res == [Token("'foo\\", type='string')] res = list(t.tokenize("bar'")) assert res == [Token("bar'", type='string')] res = list(t.tokenize("bar")) assert res == [Token('bar', type='word')] res = list(t.tokenize('"foo\\bar"')) assert res == [Token('"foo\\bar"', type="string")]
def test_string_multiline(self): t = Tokenizer(PythonSchema) res = list(t.tokenize('"""foo\n')) assert res == [Token('"""foo\n', type='string')] res = list(t.tokenize('bar\n')) assert res == [Token('bar\n', type='string')] res = list(t.tokenize('"""\n')) assert res == [Token('"""', type='string'), Token('\n', type='whitespace')] # tricky problem: the following line must not put the tokenizer in # 'multiline state'... res = list(t.tokenize('"""foo"""')) assert res == [Token('"""foo"""', type='string')] res = list(t.tokenize('bar')) assert res == [Token('bar', type='word')]
def test_something_strange(self): t = Tokenizer(PythonSchema) tokens = list(t.tokenize('"""foo "bar" baz"""')) assert not t._inside_multiline
def tokens(self, data): t = Tokenizer(PythonSchema) return list(t.tokenize(data))