def test_pipeline():
RULE = rule(pipeline(['a b c', 'b c']), 'd')
parser = Parser(RULE)
assert parser.match('b c d')
assert parser.match('a b c d')
RULE = rule(pipeline(['a b']).repeatable(), 'c')
parser = Parser(RULE)
assert parser.match('a b a b c')
RULE = rule(caseless_pipeline(['A B']), 'c')
parser = Parser(RULE)
assert parser.match('A b c')
RULE = morph_pipeline([
'текст',
'текст песни',
'материал',
'информационный материал',
])
parser = Parser(RULE)
matches = list(parser.findall('текстом песни музыкальной группы'))
assert len(matches) == 1
match = matches[0]
assert [_.value for _ in match.tokens] == ['текстом', 'песни']
matches = list(parser.findall('информационного материала под названием'))
assert len(matches) == 1
match = matches[0]
assert [_.value for _ in match.tokens] == ['информационного', 'материала']
RULE = morph_pipeline(['1 B.'])
parser = Parser(RULE)
assert parser.match('1 b .')
def test_type_errors():
F = fact('F', ['a'])
RULE = rule(
'a',
eq('1').interpretation(
custom(int)
)
).interpretation(
F.a
)
parser = Parser(RULE)
match = parser.match('a 1')
with pytest.raises(TypeError):
match.fact
F = fact('F', ['a'])
RULE = rule(
'a',
eq('1').interpretation(
custom(int)
)
).interpretation(
custom(str)
)
parser = Parser(RULE)
match = parser.match('a 1')
with pytest.raises(TypeError):
match.fact
def test_person():
Name = fact(
'Name',
['first', 'last'],
)
Person = fact(
'Person',
['position', 'name']
)
LAST = and_(
gram('Surn'),
not_(gram('Abbr')),
)
FIRST = and_(
gram('Name'),
not_(gram('Abbr')),
)
POSITION = morph_pipeline([
'управляющий директор',
'вице-мэр'
])
gnc = gnc_relation()
NAME = rule(
FIRST.interpretation(
Name.first
).match(gnc),
LAST.interpretation(
Name.last
).match(gnc)
).interpretation(
Name
)
PERSON = rule(
POSITION.interpretation(
Person.position
).match(gnc),
NAME.interpretation(
Person.name
)
).interpretation(
Person
)
parser = Parser(PERSON)
match = parser.match('управляющий директор Иван Ульянов')
assert match
assert match.fact == Person(
position='управляющий директор',
name=Name(
first='Иван',
last='Ульянов'
)
)
def visit_MinBoundedRule(self, item):
child = self.visit(item.rule)
items = repeat(child, item.min - 1)
items.append(repeatable(child, item.reverse))
from parser.parser import rule
return rule(*items)
def test_repeatable_optional():
A = rule('a')
assert_bnf(
A.optional().repeatable(),
"R0 -> e | 'a' R0 | 'a'",
)
assert_bnf(
A.repeatable().optional(),
"R0 -> e | 'a' R0 | 'a'",
)
assert_bnf(
A.repeatable().optional().repeatable(),
"R0 -> e | 'a' R0 | 'a'",
)
assert_bnf(A.repeatable().repeatable(), "R0 -> 'a' R0 | 'a'")
assert_bnf(
A.optional().optional(),
"R0 -> e | 'a'",
)
assert_bnf(A.repeatable(max=2).repeatable(), "R0 -> 'a' R0 | 'a'")
assert_bnf(A.repeatable().repeatable(min=1, max=2), "R0 -> 'a' R0 | 'a'")
assert_bnf(A.optional().repeatable(max=2), 'R0 -> e | R1',
"R1 -> 'a' 'a' | 'a'")
assert_bnf(A.repeatable(reverse=True).optional(), "R0 -> e | 'a' | 'a' R0")
assert_bnf(A.repeatable().repeatable(reverse=True), "R0 -> 'a' | 'a' R0")
assert_bnf(
A.repeatable(reverse=True).repeatable(min=1, max=2),
"R0 -> 'a' | 'a' R0")
assert_bnf(A.repeatable().repeatable(min=2, reverse=True),
"R0 -> 'a' R0 | 'a'")
assert_bnf(A.repeatable(max=2, reverse=True), "R0 -> 'a' | 'a' 'a'")
def test_merge_facts():
F = fact('F', ['a', 'b'])
A = rule(
eq('a').interpretation(F.a)
).interpretation(F)
B = rule(
eq('b').interpretation(F.b)
).interpretation(F)
RULE = rule(
A, B
).interpretation(F)
parser = Parser(RULE)
match = parser.match('a b')
record = match.fact
assert record == F(a='a', b='b')
assert record.spans == [(0, 1), (2, 3)]
assert record.as_json == {'a': 'a', 'b': 'b'}
def visit_MinMaxBoundedRule(self, item):
rule, min, max, reverse = item
child = self.visit(rule)
items = repeat(child, min - 1)
items.append(max_bound(child, max - min + 1, reverse))
from parser.parser import rule
return rule(*items)
def test_const():
RULE = rule(
'a'
).interpretation(
const(1)
)
parser = Parser(RULE)
match = parser.match('a')
assert match.fact == 1
def test_inflected():
RULE = rule(
'московским'
).interpretation(
inflected({'nomn', 'femn'})
)
parser = Parser(RULE)
match = parser.match('московским')
assert match.fact == 'московская'
def text_normalized():
RULE = rule(
'московским'
).interpretation(
normalized()
)
parser = Parser(RULE)
match = parser.match('московским')
assert match.fact == 'московский'
def test_rule_custom():
RULE = rule(
'3', '.', '14'
).interpretation(
custom(float)
)
parser = Parser(RULE)
match = parser.match('3.14')
assert match.fact == 3.14
def test_main():
relation = and_(number_relation(), gender_relation())
A = rule(gram('Surn'), main(gram('Name'))).match(relation)
B = gram('VERB').match(relation)
AB = rule(A, B)
parser = Parser(AB)
match = parser.match('иванов иван стал')
assert match
match = parser.match('иванов иван стали')
assert not match
match = parser.match('ивановы иван стал')
assert match
def repeatable(item, reverse=False):
from parser.parser import forward, or_, rule
temp = forward()
a = rule(item, temp)
b = item
if reverse:
a, b = b, a
return temp.define(or_(a, b))
def test_rule_custom_custom():
MAPPING = {'a': 1}
RULE = rule(
'A'
).interpretation(
custom(str.lower).custom(MAPPING.get)
)
parser = Parser(RULE)
match = parser.match('A')
assert match.fact == 1
def test_attribute_const():
F = fact('F', 'a')
RULE = rule(
'январь'
).interpretation(
F.a.const(1)
)
parser = Parser(RULE)
match = parser.match('январь')
assert match.fact == 1
def test_attribute():
F = fact('F', 'a')
RULE = rule(
'a'
).interpretation(
F.a
)
parser = Parser(RULE)
match = parser.match('a')
assert match.fact == 'a'
def test_predicate_attribute():
F = fact('F', ['a'])
RULE = rule(
eq('a').interpretation(F.a)
).interpretation(F)
parser = Parser(RULE)
match = parser.match('a')
record = match.fact
assert record == F(a='a')
assert record.spans == [(0, 1)]
assert record.as_json == {'a': 'a'}
def max_bound(item, count, reverse=False):
from parser.parser import rule, or_
if count == 1:
return item
else:
a = rule(item, max_bound(item, count - 1, reverse))
b = item
if reverse:
a, b = b, a
return or_(a, b)
def test_rule_attribute_custom():
F = fact('F', ['a'])
RULE = rule(
'1'
).interpretation(
F.a
).interpretation(
custom(int)
)
parser = Parser(RULE)
match = parser.match('1')
assert match.fact == 1
def test_inflected_custom():
MONTHS = {
'январь': 1
}
RULE = rule(
'январе'
).interpretation(
inflected({'nomn', 'sing'}).custom(MONTHS.get)
)
parser = Parser(RULE)
match = parser.match('январе')
assert match.fact == 1
def test_normalized_custom():
MONTHS = {
'январь': 1
}
RULE = rule(
'январе'
).interpretation(
normalized().custom(MONTHS.get)
)
parser = Parser(RULE)
match = parser.match('январе')
assert match.fact == 1
def test_insted_attributes():
F = fact('F', ['a', 'b'])
RULE = rule(
eq('a').interpretation(F.a)
).interpretation(
F.b
).interpretation(F)
parser = Parser(RULE)
match = parser.match('a')
record = match.fact
assert record == F(a=None, b='a')
assert record.spans == [(0, 1)]
assert record.as_json == {'b': 'a'}
def test_bounded():
A = rule('a')
with pytest.raises(ValueError):
A.repeatable(min=-1)
with pytest.raises(ValueError):
A.repeatable(min=-1)
with pytest.raises(ValueError):
A.repeatable(min=2, max=1)
assert_bnf(A.repeatable(max=3), "R0 -> 'a' R1 | 'a'",
"R1 -> 'a' 'a' | 'a'")
assert_bnf(A.repeatable(min=2), "R0 -> 'a' R1", "R1 -> 'a' R1 | 'a'")
assert_bnf(A.repeatable(min=2, max=3), "R0 -> 'a' R1",
"R1 -> 'a' 'a' | 'a'")
def repeatable_optional(item,
reverse_repeatable=False,
reverse_optional=False):
from parser.parser import forward, or_, rule, empty
temp = forward()
a = empty()
b = rule(item, temp)
c = item
if reverse_repeatable:
b, c = c, b
if reverse_optional:
a, b, c = b, c, a
return temp.define(or_(a, b, c))
def test_repeatable():
F = fact('F', [attribute('a').repeatable()])
RULE = rule(
eq('a').interpretation(F.a),
eq('b').interpretation(F.a)
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('a b')
record = match.fact
assert record == F(a=['a', 'b'])
assert record.spans == [(0, 1), (2, 3)]
assert record.as_json == {'a': ['a', 'b']}
def test_attribute_custom_custom():
F = fact('F', 'a')
MAPPING = {'a': 1}
RULE = rule(
'A'
).interpretation(
F.a.custom(str.lower).custom(MAPPING.get)
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('A')
record = match.fact
assert record == F(a=1)
def test_attribute_custom():
F = fact('F', 'a')
RULE = rule(
'1'
).interpretation(
F.a.custom(int)
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('1')
record = match.fact
assert record == F(a=1)
assert record.spans == [(0, 1)]
assert record.as_json == {'a': 1}
def test_normalized_custom_attribute():
F = fact('F', ['a'])
MONTHS = {
'январь': 1
}
RULE = rule(
'январе'
).interpretation(
F.a.normalized().custom(MONTHS.get)
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('январе')
assert match.fact == F(a=1)
def test_attribute_inflected():
F = fact('F', 'a')
RULE = rule(
'январе'
).interpretation(
F.a.inflected({'nomn', 'plur'})
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('январе')
record = match.fact
assert record == F(a='январи')
assert record.spans == [(0, 6)]
assert record.as_json == {'a': 'январи'}
def test_attribute_normalized():
F = fact('F', 'a')
RULE = rule(
'январе'
).interpretation(
F.a.normalized()
).interpretation(
F
)
parser = Parser(RULE)
match = parser.match('январе')
record = match.fact
assert record == F(a='январь')
assert record.spans == [(0, 6)]
assert record.as_json == {'a': 'январь'}
