def test_tokenize_spec(self):
"""Tokenize tagset specs"""
self.assertEqual(list(tokenize(r"~(a | b) \ c")),
["∁", "(", "a", "∪", "b", ")", "∖", "c"])
self.assertEqual(list(tokenize("a ∩ b ∪ ∁ c")),
["a", "∩", "b", "∪", "∁", "c"])
def test_unbalanced_parens(self):
"""Unbalanced parenthesis"""
tokens = tokenize("((a)")
self.assertEqual(tokens.next(), "(")
self.assertEqual(tokens.next(), "(")
self.assertEqual(tokens.next(), "a")
self.assertEqual(tokens.next(), ")")
self.assertRaises(SpecSyntaxError, tokens.next)
def test_invalid_token(self):
"""Invalid token"""
tokens = tokenize("a + b")
self.assertEqual(tokens.next(), "a")
self.assertRaises(SpecSyntaxError, tokens.next)
def test_tag(self):
"""Tokenize tags"""
self.assertEqual(list(tokenize("foo-bar báz quüx")),
["foo-bar", "báz", "quüx"])
