def test_grammar_choice_dict_mismatch(self): grammar = Grammar(ChoiceDict(Sequence('NUMBER'), Sequence('WORD'))) tokens = tokenize([(',', ',', 3)]) with self.assertRaises(textparser.Error) as cm: grammar.parse(tokens) self.assertEqual(cm.exception.offset, 3)
def test_grammar_sequence_mismatch(self): grammar = Grammar(Sequence('NUMBER', 'WORD')) tokens = tokenize([('NUMBER', '1.45')]) with self.assertRaises(textparser.GrammarError) as cm: grammar.parse(tokens) self.assertEqual(cm.exception.offset, -1)
def test_grammar_sequence(self): grammar = Grammar(Sequence('NUMBER', 'WORD')) tokens = tokenize([ ('NUMBER', '1.45'), ('WORD', 'm') ]) tree = grammar.parse(tokens) self.assertEqual(tree, ['1.45', 'm'])
def test_grammar_error(self): grammar = Grammar(NoMatch()) datas = [[('NUMBER', '1', 3)], [('WORD', 'foo', 3)]] for tokens in datas: tokens = tokenize(tokens) with self.assertRaises(textparser.GrammarError) as cm: grammar.parse(tokens) self.assertEqual(cm.exception.offset, 3) self.assertEqual(str(cm.exception), 'Invalid syntax at offset 3.')
def test_grammar_optional(self): grammar = Grammar(Sequence(Optional('WORD'), Optional('WORD'), Optional('NUMBER'))) datas = [ ( [], [[], [], []] ), ( [('WORD', 'a')], [['a'], [], []] ), ( [('NUMBER', 'c')], [[], [], ['c']] ), ( [('WORD', 'a'), ('NUMBER', 'c')], [['a'], [], ['c']] ), ( [('WORD', 'a'), ('WORD', 'b'), ('NUMBER', 'c')], [['a'], ['b'], ['c']] ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_1(self): grammar = Grammar(Sequence( 'IF', choice(Sequence(choice('A', 'B'), 'STRING'), 'STRING'), 'WORD', choice( Sequence( choice(DelimitedList('STRING'), ZeroOrMore('NUMBER')), '.'), '.'))) datas = [ ( [ ('IF', 'IF'), ('STRING', 'foo'), ('WORD', 'bar'), ('.', '.') ], ['IF', 'foo', 'bar', [[], '.']] ), ( [ ('IF', 'IF'), ('STRING', 'foo'), ('WORD', 'bar'), ('NUMBER', '0'), ('NUMBER', '100'), ('.', '.') ], ['IF', 'foo', 'bar', [['0', '100'], '.']] ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more_dict_mismatch(self): grammar = Grammar(OneOrMoreDict(Sequence('WORD', 'NUMBER'))) datas = [ ( [('WORD', 'foo', 5)], -1 ), ( [ ('WORD', 'foo', 5), ('WORD', 'bar', 6) ], 6 ), ( [ ('WORD', 'foo', 5), ('NUMBER', '4', 6), ('WORD', 'bar', 7), ('WORD', 'fie', 8) ], 8 ) ] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice_mismatch(self): grammar = Grammar(Choice(Sequence('NUMBER', 'WORD'), 'WORD')) datas = [([('NUMBER', '1', 5)], -1), ([('NUMBER', '1', 5), ('NUMBER', '2', 7)], 7)] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice_dict(self): number = Forward() number <<= Sequence('NUMBER') grammar = Grammar(ChoiceDict(number, Tag('foo', Sequence('WORD')), ChoiceDict('BAR'), 'FIE')) datas = [ ( [('WORD', 'm')], ('foo', ['m']) ), ( [('NUMBER', '5')], ['5'] ), ( [('BAR', 'foo')], 'foo' ), ( [('FIE', 'fum')], 'fum' ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more(self): grammar = Grammar(OneOrMore('WORD')) datas = [([('WORD', 'foo')], ['foo']), ([('WORD', 'foo'), ('WORD', 'bar')], ['foo', 'bar'])] self.parse_and_assert_tree(grammar, datas)
def test_grammar_any_until(self): grammar = Grammar(Sequence(AnyUntil('STRING'), 'STRING')) datas = [([('NUMBER', '1'), ('WORD', 'a'), ('STRING', '"b"')], [['1', 'a'], '"b"'])] self.parse_and_assert_tree(grammar, datas)
def test_grammar_forward(self): foo = Forward() foo <<= Sequence('FOO') grammar = Grammar(foo) datas = [([('FOO', 'foo')], ['foo'])] self.parse_and_assert_tree(grammar, datas)
def test_grammar_forward_text(self): foo = Forward() foo <<= 'FOO' grammar = Grammar(foo) datas = [([('FOO', 'foo')], 'foo')] self.parse_and_assert_tree(grammar, datas)
def test_grammar_delimited_list_mismatch(self): grammar = Grammar(Sequence(DelimitedList('WORD'), Optional('.'))) datas = [([('WORD', 'foo', 1), (',', ',', 2)], 2), ([('WORD', 'foo', 1), (',', ',', 2), ('WORD', 'foo', 3), (',', ',', 4), ('.', '.', 5)], 4)] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_zero_or_more_partial_element_match(self): grammar = Grammar( Sequence(ZeroOrMore(Sequence('WORD', 'NUMBER')), 'WORD')) datas = [([('WORD', 'foo'), ('NUMBER', '1'), ('WORD', 'bar'), ('NUMBER', '2'), ('WORD', 'fie')], [[['foo', '1'], ['bar', '2']], 'fie'])] self.parse_and_assert_tree(grammar, datas)
def test_grammar_delimited_list(self): grammar = Grammar(Sequence(DelimitedList('WORD'), Optional('.'))) datas = [([('WORD', 'foo')], [['foo'], []]), ([('WORD', 'foo'), (',', ','), ('WORD', 'bar')], [['foo', 'bar'], []]), ([('WORD', 'foo'), (',', ','), ('WORD', 'bar'), ('.', '.')], [['foo', 'bar'], ['.']])] self.parse_and_assert_tree(grammar, datas)
def test_grammar_not(self): grammar = Grammar(Sequence(Not('WORD'), 'NUMBER')) datas = [ ( [('NUMBER', '1')], [[], '1'] ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_not_mismatch(self): grammar = Grammar(Sequence(Not('WORD'), 'NUMBER')) datas = [ ( [('WORD', 'foo', 3), ('NUMBER', '1', 4)], 3 ) ] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_tag_mismatch(self): grammar = Grammar(Tag('a', 'WORD')) datas = [ ( [('NUMBER', 'bar')], 1 ) ] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_zero_or_more_dict(self): grammar = Grammar(ZeroOrMoreDict(Sequence('WORD', 'NUMBER'))) datas = [([], {}), ([('WORD', 'foo'), ('NUMBER', '1'), ('WORD', 'bar'), ('NUMBER', '2'), ('WORD', 'foo'), ('NUMBER', '3')], { 'foo': [['foo', '1'], ['foo', '3']], 'bar': [['bar', '2']] })] self.parse_and_assert_tree(grammar, datas)
def test_grammar_none(self): class AnyAsNone(textparser.Pattern): def match(self, tokens): tokens.get_value() return None grammar = Grammar(AnyAsNone()) datas = [([('NUMBER', '1')], None)] self.parse_and_assert_tree(grammar, datas)
def test_grammar_tag(self): grammar = Grammar( Tag( 'a', Tag('b', choice(Tag('c', 'WORD'), Tag('d', Optional('NUMBER')))))) datas = [([('WORD', 'bar')], ('a', ('b', ('c', 'bar')))), ([('NUMBER', '1')], ('a', ('b', ('d', ['1'])))), ([], ('a', ('b', ('d', []))))] self.parse_and_assert_tree(grammar, datas)
def test_grammar_1_mismatch(self): grammar = Grammar(Sequence( 'IF', choice(Sequence(choice('A', 'B'), 'STRING'), 'STRING'), 'WORD', choice( Sequence( choice(DelimitedList('STRING'), ZeroOrMore('NUMBER')), '.'), '.'))) datas = [ ( [ ('IF', 'IF', 1), ('STRING', 'foo', 2), ('WORD', 'bar', 3), (',', ',', 4) ], 4 ), ( [ ('IF', 'IF', 1), ('STRING', 'foo', 2), ('.', '.', 3) ], 3 ), ( [ ('IF', 'IF', 1), ('NUMBER', '1', 2) ], 2 ), ( [ ('IF', 'IF', 1), ('STRING', 'foo', 2), ('WORD', 'bar', 3), ('.', '.', 4), ('.', '.', 5) ], 5 ) ] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_any(self): grammar = Grammar(Any()) datas = [ ( [('A', r'a')], 'a' ), ( [('B', r'b')], 'b' ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_no_match(self): grammar = Grammar(NoMatch()) datas = [ ( [('NUMBER', '1', 3)], 3 ), ( [('WORD', 'foo', 3)], 3 ) ] self.parse_and_assert_mismatch(grammar, datas)
def test_grammar_choice(self): grammar = Grammar(Choice('NUMBER', 'WORD')) datas = [ ( [('WORD', 'm')], 'm' ), ( [('NUMBER', '5')], '5' ) ] self.parse_and_assert_tree(grammar, datas)
def test_grammar_one_or_more_mismatch(self): grammar = Grammar(OneOrMore('WORD')) datas = [ ( [] , -1 ), ( [('NUMBER', 'foo', 2)], 2 ) ] self.parse_and_assert_mismatch(grammar, datas)
def grammar(self): return Grammar(Sequence('NUMBER', 'WORD'))
def grammar(self): return Grammar('NUMBER')