def test_grammar_choice_dict_mismatch(self):
grammar = Grammar(ChoiceDict(Sequence('NUMBER'), Sequence('WORD')))
tokens = tokenize([(',', ',', 3)])
with self.assertRaises(textparser.Error) as cm:
grammar.parse(tokens)
self.assertEqual(cm.exception.offset, 3)
def test_grammar_sequence_mismatch(self):
grammar = Grammar(Sequence('NUMBER', 'WORD'))
tokens = tokenize([('NUMBER', '1.45')])
with self.assertRaises(textparser.GrammarError) as cm:
grammar.parse(tokens)
self.assertEqual(cm.exception.offset, -1)
def test_grammar_sequence(self):
grammar = Grammar(Sequence('NUMBER', 'WORD'))
tokens = tokenize([
('NUMBER', '1.45'),
('WORD', 'm')
])
tree = grammar.parse(tokens)
self.assertEqual(tree, ['1.45', 'm'])
def test_grammar_error(self):
grammar = Grammar(NoMatch())
datas = [[('NUMBER', '1', 3)], [('WORD', 'foo', 3)]]
for tokens in datas:
tokens = tokenize(tokens)
with self.assertRaises(textparser.GrammarError) as cm:
grammar.parse(tokens)
self.assertEqual(cm.exception.offset, 3)
self.assertEqual(str(cm.exception), 'Invalid syntax at offset 3.')
def test_grammar_optional(self):
grammar = Grammar(Sequence(Optional('WORD'),
Optional('WORD'),
Optional('NUMBER')))
datas = [
(
[],
[[], [], []]
),
(
[('WORD', 'a')],
[['a'], [], []]
),
(
[('NUMBER', 'c')],
[[], [], ['c']]
),
(
[('WORD', 'a'), ('NUMBER', 'c')],
[['a'], [], ['c']]
),
(
[('WORD', 'a'), ('WORD', 'b'), ('NUMBER', 'c')],
[['a'], ['b'], ['c']]
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_1(self):
grammar = Grammar(Sequence(
'IF',
choice(Sequence(choice('A', 'B'), 'STRING'),
'STRING'),
'WORD',
choice(
Sequence(
choice(DelimitedList('STRING'), ZeroOrMore('NUMBER')), '.'),
'.')))
datas = [
(
[
('IF', 'IF'),
('STRING', 'foo'),
('WORD', 'bar'),
('.', '.')
],
['IF', 'foo', 'bar', [[], '.']]
),
(
[
('IF', 'IF'),
('STRING', 'foo'),
('WORD', 'bar'),
('NUMBER', '0'),
('NUMBER', '100'),
('.', '.')
],
['IF', 'foo', 'bar', [['0', '100'], '.']]
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more_dict_mismatch(self):
grammar = Grammar(OneOrMoreDict(Sequence('WORD', 'NUMBER')))
datas = [
(
[('WORD', 'foo', 5)],
-1
),
(
[
('WORD', 'foo', 5),
('WORD', 'bar', 6)
],
6
),
(
[
('WORD', 'foo', 5),
('NUMBER', '4', 6),
('WORD', 'bar', 7),
('WORD', 'fie', 8)
],
8
)
]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice_mismatch(self):
grammar = Grammar(Choice(Sequence('NUMBER', 'WORD'), 'WORD'))
datas = [([('NUMBER', '1', 5)], -1),
([('NUMBER', '1', 5), ('NUMBER', '2', 7)], 7)]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice_dict(self):
number = Forward()
number <<= Sequence('NUMBER')
grammar = Grammar(ChoiceDict(number,
Tag('foo', Sequence('WORD')),
ChoiceDict('BAR'),
'FIE'))
datas = [
(
[('WORD', 'm')],
('foo', ['m'])
),
(
[('NUMBER', '5')],
['5']
),
(
[('BAR', 'foo')],
'foo'
),
(
[('FIE', 'fum')],
'fum'
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more(self):
grammar = Grammar(OneOrMore('WORD'))
datas = [([('WORD', 'foo')], ['foo']),
([('WORD', 'foo'), ('WORD', 'bar')], ['foo', 'bar'])]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_any_until(self):
grammar = Grammar(Sequence(AnyUntil('STRING'), 'STRING'))
datas = [([('NUMBER', '1'), ('WORD', 'a'),
('STRING', '"b"')], [['1', 'a'], '"b"'])]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_forward(self):
foo = Forward()
foo <<= Sequence('FOO')
grammar = Grammar(foo)
datas = [([('FOO', 'foo')], ['foo'])]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_forward_text(self):
foo = Forward()
foo <<= 'FOO'
grammar = Grammar(foo)
datas = [([('FOO', 'foo')], 'foo')]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_delimited_list_mismatch(self):
grammar = Grammar(Sequence(DelimitedList('WORD'), Optional('.')))
datas = [([('WORD', 'foo', 1), (',', ',', 2)], 2),
([('WORD', 'foo', 1), (',', ',', 2), ('WORD', 'foo', 3),
(',', ',', 4), ('.', '.', 5)], 4)]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_zero_or_more_partial_element_match(self):
grammar = Grammar(
Sequence(ZeroOrMore(Sequence('WORD', 'NUMBER')), 'WORD'))
datas = [([('WORD', 'foo'), ('NUMBER', '1'), ('WORD', 'bar'),
('NUMBER', '2'), ('WORD', 'fie')], [[['foo', '1'],
['bar', '2']], 'fie'])]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_delimited_list(self):
grammar = Grammar(Sequence(DelimitedList('WORD'), Optional('.')))
datas = [([('WORD', 'foo')], [['foo'], []]),
([('WORD', 'foo'), (',', ','),
('WORD', 'bar')], [['foo', 'bar'], []]),
([('WORD', 'foo'), (',', ','), ('WORD', 'bar'),
('.', '.')], [['foo', 'bar'], ['.']])]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_not(self):
grammar = Grammar(Sequence(Not('WORD'), 'NUMBER'))
datas = [
(
[('NUMBER', '1')],
[[], '1']
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_not_mismatch(self):
grammar = Grammar(Sequence(Not('WORD'), 'NUMBER'))
datas = [
(
[('WORD', 'foo', 3), ('NUMBER', '1', 4)],
3
)
]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_tag_mismatch(self):
grammar = Grammar(Tag('a', 'WORD'))
datas = [
(
[('NUMBER', 'bar')],
1
)
]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_zero_or_more_dict(self):
grammar = Grammar(ZeroOrMoreDict(Sequence('WORD', 'NUMBER')))
datas = [([], {}),
([('WORD', 'foo'), ('NUMBER', '1'), ('WORD', 'bar'),
('NUMBER', '2'), ('WORD', 'foo'), ('NUMBER', '3')], {
'foo': [['foo', '1'], ['foo', '3']],
'bar': [['bar', '2']]
})]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_none(self):
class AnyAsNone(textparser.Pattern):
def match(self, tokens):
tokens.get_value()
return None
grammar = Grammar(AnyAsNone())
datas = [([('NUMBER', '1')], None)]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_tag(self):
grammar = Grammar(
Tag(
'a',
Tag('b', choice(Tag('c', 'WORD'), Tag('d',
Optional('NUMBER'))))))
datas = [([('WORD', 'bar')], ('a', ('b', ('c', 'bar')))),
([('NUMBER', '1')], ('a', ('b', ('d', ['1'])))),
([], ('a', ('b', ('d', []))))]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_1_mismatch(self):
grammar = Grammar(Sequence(
'IF',
choice(Sequence(choice('A', 'B'), 'STRING'),
'STRING'),
'WORD',
choice(
Sequence(
choice(DelimitedList('STRING'), ZeroOrMore('NUMBER')), '.'),
'.')))
datas = [
(
[
('IF', 'IF', 1),
('STRING', 'foo', 2),
('WORD', 'bar', 3),
(',', ',', 4)
],
4
),
(
[
('IF', 'IF', 1),
('STRING', 'foo', 2),
('.', '.', 3)
],
3
),
(
[
('IF', 'IF', 1),
('NUMBER', '1', 2)
],
2
),
(
[
('IF', 'IF', 1),
('STRING', 'foo', 2),
('WORD', 'bar', 3),
('.', '.', 4),
('.', '.', 5)
],
5
)
]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_any(self):
grammar = Grammar(Any())
datas = [
(
[('A', r'a')],
'a'
),
(
[('B', r'b')],
'b'
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_no_match(self):
grammar = Grammar(NoMatch())
datas = [
(
[('NUMBER', '1', 3)],
3
),
(
[('WORD', 'foo', 3)],
3
)
]
self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice(self):
grammar = Grammar(Choice('NUMBER', 'WORD'))
datas = [
(
[('WORD', 'm')],
'm'
),
(
[('NUMBER', '5')],
'5'
)
]
self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more_mismatch(self):
grammar = Grammar(OneOrMore('WORD'))
datas = [
(
[]
, -1
),
(
[('NUMBER', 'foo', 2)],
2
)
]
self.parse_and_assert_mismatch(grammar, datas)
def grammar(self):
return Grammar(Sequence('NUMBER', 'WORD'))
def grammar(self):
return Grammar('NUMBER')
