def test_get_tokens_with_errors(self):
parser = Parser()
with self.assertRaises(ParseError) as cm:
parser.get_tokens('\\')
self.assertEqual(
u"Expect symbol after backslash, pos=1",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[1-3]asd\\')
self.assertEqual(
u"Expect symbol after backslash, pos=9",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][asd-89]asd\\')
self.assertEqual(
u"Expect number after '[', pos=10",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][78-qwe]asd\\')
self.assertEqual(
u"Expect number after '-', pos=13",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][78-9]\\')
self.assertEqual(
u"Expect symbol after backslash, pos=16",
cm.exception.message)
def test_get_tokens(self):
parser = Parser()
tokens = parser.get_tokens('')
self.assertEqual([], tokens)
tokens = parser.get_tokens(r'asd[1990] [2] #^&\ [33-44]')
expect_tokens = [
Token(Token.STR, 'asd'),
Token(Token.LBRS),
Token(Token.NUM, '1990'),
Token(Token.RBRS),
Token(Token.STR, ' '),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.STR, ' #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
]
self.assertEqual(expect_tokens, tokens)
tokens = parser.get_tokens(r'[12-345]\]-. ]!wer][2][3-4]- #^&\ [33-44]]')
expect_tokens = [
Token(Token.LBRS),
Token(Token.NUM, '12'),
Token(Token.MINUS),
Token(Token.NUM, '345'),
Token(Token.RBRS),
Token(Token.STR, ']-. ]!wer]'),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.LBRS),
Token(Token.NUM, '3'),
Token(Token.MINUS),
Token(Token.NUM, '4'),
Token(Token.RBRS),
Token(Token.STR, '- #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
Token(Token.STR, ']'),
]
self.assertEqual(expect_tokens, tokens)
tokens = parser.get_tokens(ur'\\\\[12-345]]-. ]!wer][2][3-4]- #^&\ [33-44]]')
expect_tokens = [
Token(Token.STR, '\\\\'),
Token(Token.LBRS),
Token(Token.NUM, '12'),
Token(Token.MINUS),
Token(Token.NUM, '345'),
Token(Token.RBRS),
Token(Token.STR, ']-. ]!wer]'),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.LBRS),
Token(Token.NUM, '3'),
Token(Token.MINUS),
Token(Token.NUM, '4'),
Token(Token.RBRS),
Token(Token.STR, '- #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
Token(Token.STR, ']'),
]
self.assertEqual(expect_tokens, tokens)
