def test_to_tokens_good(self):
eq_(to_tokens(u'abc'),
[u'abc'])
eq_(to_tokens(u'abc def'),
[u'abc', u'def'])
eq_(to_tokens(u'abc OR "def"'),
[u'abc', u'OR', u'"def"'])
eq_(to_tokens(u'abc OR "def ghi"'),
[u'abc', u'OR', u'"def ghi"'])
eq_(to_tokens(u'abc AND "def ghi"'),
[u'abc', u'AND', u'"def ghi"'])
def test_escaping(self):
"""Escaped things stay escaped"""
tests = [
(u'\\"AND', [u'\\"AND']),
(u'\\"AND\\"', [u'\\"AND\\"']),
]
for text, expected in tests:
eq_(to_tokens(text), expected)
def test_to_tokens_edge_cases(self):
eq_(to_tokens(u'AND "def ghi'),
[u'AND', u'"def ghi"'])
