def test_get_tokens_with_errors(self): parser = Parser() with self.assertRaises(ParseError) as cm: parser.get_tokens('\\') self.assertEqual( u"Expect symbol after backslash, pos=1", cm.exception.message) with self.assertRaises(ParseError) as cm: parser.get_tokens('[1-3]asd\\') self.assertEqual( u"Expect symbol after backslash, pos=9", cm.exception.message) with self.assertRaises(ParseError) as cm: parser.get_tokens('[123-456][asd-89]asd\\') self.assertEqual( u"Expect number after '[', pos=10", cm.exception.message) with self.assertRaises(ParseError) as cm: parser.get_tokens('[123-456][78-qwe]asd\\') self.assertEqual( u"Expect number after '-', pos=13", cm.exception.message) with self.assertRaises(ParseError) as cm: parser.get_tokens('[123-456][78-9]\\') self.assertEqual( u"Expect symbol after backslash, pos=16", cm.exception.message)
def test_get_tokens(self): parser = Parser() tokens = parser.get_tokens('') self.assertEqual([], tokens) tokens = parser.get_tokens(r'asd[1990] [2] #^&\ [33-44]') expect_tokens = [ Token(Token.STR, 'asd'), Token(Token.LBRS), Token(Token.NUM, '1990'), Token(Token.RBRS), Token(Token.STR, ' '), Token(Token.LBRS), Token(Token.NUM, '2'), Token(Token.RBRS), Token(Token.STR, ' #^& '), Token(Token.LBRS), Token(Token.NUM, '33'), Token(Token.MINUS), Token(Token.NUM, '44'), Token(Token.RBRS), ] self.assertEqual(expect_tokens, tokens) tokens = parser.get_tokens(r'[12-345]\]-. ]!wer][2][3-4]- #^&\ [33-44]]') expect_tokens = [ Token(Token.LBRS), Token(Token.NUM, '12'), Token(Token.MINUS), Token(Token.NUM, '345'), Token(Token.RBRS), Token(Token.STR, ']-. ]!wer]'), Token(Token.LBRS), Token(Token.NUM, '2'), Token(Token.RBRS), Token(Token.LBRS), Token(Token.NUM, '3'), Token(Token.MINUS), Token(Token.NUM, '4'), Token(Token.RBRS), Token(Token.STR, '- #^& '), Token(Token.LBRS), Token(Token.NUM, '33'), Token(Token.MINUS), Token(Token.NUM, '44'), Token(Token.RBRS), Token(Token.STR, ']'), ] self.assertEqual(expect_tokens, tokens) tokens = parser.get_tokens(ur'\\\\[12-345]]-. ]!wer][2][3-4]- #^&\ [33-44]]') expect_tokens = [ Token(Token.STR, '\\\\'), Token(Token.LBRS), Token(Token.NUM, '12'), Token(Token.MINUS), Token(Token.NUM, '345'), Token(Token.RBRS), Token(Token.STR, ']-. ]!wer]'), Token(Token.LBRS), Token(Token.NUM, '2'), Token(Token.RBRS), Token(Token.LBRS), Token(Token.NUM, '3'), Token(Token.MINUS), Token(Token.NUM, '4'), Token(Token.RBRS), Token(Token.STR, '- #^& '), Token(Token.LBRS), Token(Token.NUM, '33'), Token(Token.MINUS), Token(Token.NUM, '44'), Token(Token.RBRS), Token(Token.STR, ']'), ] self.assertEqual(expect_tokens, tokens)