def test_parse(self):
parser = Parser()
tokens = [
Token(Token.STR, 'asd'),
Token(Token.LBRS),
Token(Token.NUM, '1990'),
Token(Token.RBRS),
Token(Token.STR, ' '),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.STR, ' #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
]
expected_nodes = [
Node(Node.STR, value='asd'),
Node(Node.SLICE, stop=1990),
Node(Node.STR, value=' '),
Node(Node.SLICE, stop=2),
Node(Node.STR, value=' #^& '),
Node(Node.SLICE, start=33, stop=44),
]
nodes = parser.parse(tokens)
self.assertEqual(expected_nodes, nodes)
def test_compile(self):
parser = Parser()
nodes = [
Node(Node.STR, value='asd '),
Node(Node.SLICE, stop=3),
Node(Node.SLICE, start=1, stop=2),
]
mask_func = parser.compile(nodes)
self.assertIsInstance(mask_func, LambdaType)
s = ''.join(map(str, range(10)))
self.assertEqual(u'asd 31', mask_func(s))
def test_parse_with_error_unexpected_end(self):
parser = Parser()
test_tokens = [Token(Token.LBRS)]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(u"Unexpected end of tokens", cm.exception.message)
test_tokens = [Token(Token.LBRS), Token(Token.NUM, '1')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(u"Unexpected end of tokens", cm.exception.message)
test_tokens = [
Token(Token.LBRS), Token(Token.NUM, '1'), Token(Token.MINUS)]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(u"Unexpected end of tokens", cm.exception.message)
test_tokens = [
Token(Token.LBRS), Token(Token.NUM, '1'),
Token(Token.MINUS), Token(Token.NUM, '2')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(u"Unexpected end of tokens", cm.exception.message)
def test_get_tokens(self):
parser = Parser()
tokens = parser.get_tokens('')
self.assertEqual([], tokens)
tokens = parser.get_tokens(r'asd[1990] [2] #^&\ [33-44]')
expect_tokens = [
Token(Token.STR, 'asd'),
Token(Token.LBRS),
Token(Token.NUM, '1990'),
Token(Token.RBRS),
Token(Token.STR, ' '),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.STR, ' #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
]
self.assertEqual(expect_tokens, tokens)
tokens = parser.get_tokens(r'[12-345]\]-. ]!wer][2][3-4]- #^&\ [33-44]]')
expect_tokens = [
Token(Token.LBRS),
Token(Token.NUM, '12'),
Token(Token.MINUS),
Token(Token.NUM, '345'),
Token(Token.RBRS),
Token(Token.STR, ']-. ]!wer]'),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.LBRS),
Token(Token.NUM, '3'),
Token(Token.MINUS),
Token(Token.NUM, '4'),
Token(Token.RBRS),
Token(Token.STR, '- #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
Token(Token.STR, ']'),
]
self.assertEqual(expect_tokens, tokens)
tokens = parser.get_tokens(ur'\\\\[12-345]]-. ]!wer][2][3-4]- #^&\ [33-44]]')
expect_tokens = [
Token(Token.STR, '\\\\'),
Token(Token.LBRS),
Token(Token.NUM, '12'),
Token(Token.MINUS),
Token(Token.NUM, '345'),
Token(Token.RBRS),
Token(Token.STR, ']-. ]!wer]'),
Token(Token.LBRS),
Token(Token.NUM, '2'),
Token(Token.RBRS),
Token(Token.LBRS),
Token(Token.NUM, '3'),
Token(Token.MINUS),
Token(Token.NUM, '4'),
Token(Token.RBRS),
Token(Token.STR, '- #^& '),
Token(Token.LBRS),
Token(Token.NUM, '33'),
Token(Token.MINUS),
Token(Token.NUM, '44'),
Token(Token.RBRS),
Token(Token.STR, ']'),
]
self.assertEqual(expect_tokens, tokens)
def test_build(self):
parser = Parser()
mask_func = parser.build(ur'asd |[1]|[2-5]|qwe|[6-9]|')
s = ''.join(map(str, range(10)))
self.assertEqual(u'asd |1|234|qwe|678|', mask_func(s))
def test_parse_with_error_unexpected_token(self):
parser = Parser()
# token != Token.STR ot token != Token.LBRS
test_tokens = [Token(Token.NUM, '1')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(
u"Unexpected token Token(Number, '1') after None, pos=0",
cm.exception.message)
test_tokens = [Token(Token.LBRS), Token(Token.STR, 'a')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(
u"Unexpected token Token(String, 'a') after Token(LeftBracketSquare), pos=1",
cm.exception.message)
test_tokens = [
Token(Token.LBRS), Token(Token.NUM, '1'), Token(Token.STR, 'a')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(
u"Unexpected token Token(String, 'a') after Token(Number, '1'), pos=2",
cm.exception.message)
test_tokens = [Token(Token.LBRS), Token(Token.NUM, '1'), Token(Token.MINUS), Token(Token.STR, 'a')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(
u"Unexpected token Token(String, 'a') after Token(Minus), pos=3",
cm.exception.message)
test_tokens = [
Token(Token.LBRS), Token(Token.NUM, '1'),
Token(Token.MINUS), Token(Token.NUM, '2'), Token(Token.STR, 'a')]
with self.assertRaises(ParseError) as cm:
parser.parse(test_tokens)
self.assertEqual(
u"Unexpected token Token(String, 'a') after Token(Number, '2'), pos=4",
cm.exception.message)
def test_get_tokens_with_errors(self):
parser = Parser()
with self.assertRaises(ParseError) as cm:
parser.get_tokens('\\')
self.assertEqual(
u"Expect symbol after backslash, pos=1",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[1-3]asd\\')
self.assertEqual(
u"Expect symbol after backslash, pos=9",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][asd-89]asd\\')
self.assertEqual(
u"Expect number after '[', pos=10",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][78-qwe]asd\\')
self.assertEqual(
u"Expect number after '-', pos=13",
cm.exception.message)
with self.assertRaises(ParseError) as cm:
parser.get_tokens('[123-456][78-9]\\')
self.assertEqual(
u"Expect symbol after backslash, pos=16",
cm.exception.message)
