def test_to_tokens_good(self): eq_(to_tokens(u'abc'), [u'abc']) eq_(to_tokens(u'abc def'), [u'abc', u'def']) eq_(to_tokens(u'abc OR "def"'), [u'abc', u'OR', u'"def"']) eq_(to_tokens(u'abc OR "def ghi"'), [u'abc', u'OR', u'"def ghi"']) eq_(to_tokens(u'abc AND "def ghi"'), [u'abc', u'AND', u'"def ghi"'])
def test_escaping(self): """Escaped things stay escaped""" tests = [ (u'\\"AND', [u'\\"AND']), (u'\\"AND\\"', [u'\\"AND\\"']), ] for text, expected in tests: eq_(to_tokens(text), expected)
def test_to_tokens_edge_cases(self): eq_(to_tokens(u'AND "def ghi'), [u'AND', u'"def ghi"'])