def test_lexicon(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
self.assertEqual(lexicon.lemma_to_stems["FOO"],
[("bar", "foo", set())])
self.assertEqual(lexicon.stem_to_lemma_key_regex["foo"],
{("FOO", "bar", ())})
def setUp(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
rules = StemmingRuleSet()
self.rule = rules.add("barista", "|o><|llow")
self.inflexion = Inflexion()
self.inflexion.add_lexicon(lexicon)
self.inflexion.add_stemming_rule_set(rules)
def setUp(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
rules = StemmingRuleSet()
self.rule = rules.add("barista", "|o><|llow")
self.inflexion = Inflexion()
self.inflexion.add_lexicon(lexicon)
self.inflexion.add_stemming_rule_set(rules)
def test_find_stems_with_tags_2(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "faa", {'-a'})
lexicon.add("FOO", "bar", "fee", {'-b'})
self.assertEqual(
lexicon.find_stems("FOO", "barista"),
{"faa", "fee"}
)
def load_lexicon(lexicon_file, pre_processor=lambda x: x):
lexicon = Lexicon()
partnum_to_key_regex = {
"1-": "P",
"1-A": "PA",
"1-M": "PM",
"1+": "I",
"2-": "F[AM]",
"2-A": "FA",
"2-M": "FM",
"3-": "A[AM][NPDSO]",
"3+": "A[AM]I",
"3+A": "AAI",
"3+M": "AMI",
"4-": "XA",
"4+": "YA",
"5-": "X[MP]",
"5+": "Y[MP]",
"6-": "AP[NPDSO]",
"6+": "API",
"7-": "FP",
}
form_override = {}
accent_override = defaultdict(list)
with open(lexicon_file) as f:
for lemma, entry in yaml.load(f).items():
if "stems" not in entry:
continue
stems = []
for partnum, stems in sorted(entry["stems"].items()):
key_regex = partnum_to_key_regex[partnum]
for stem, tag in split_stem_tags(stems):
lexicon.add(lemma, key_regex, pre_processor(stem), tag)
for key_regex, stems in entry.get("stem_overrides", []):
if stems is None:
continue
for stem, tag in split_stem_tags(stems):
lexicon.add(lemma, key_regex, pre_processor(stem), tag)
for key, form in entry.get("forms", {}).items():
form_override[(lemma, key)] = form
for key_regex, form in entry.get("accents", []):
accent_override[lemma].append((key_regex, form))
return lexicon, form_override, accent_override
def test_lexicon(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
self.assertEqual(
lexicon.lemma_to_stems["FOO"],
[("bar", "foo", set())]
)
self.assertEqual(
lexicon.stem_to_lemma_key_regex["foo"],
{("FOO", "bar", ())}
)
def test_find_stems_with_tags_2(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "faa", {'-a'})
lexicon.add("FOO", "bar", "fee", {'-b'})
self.assertEqual(lexicon.find_stems("FOO", "barista"), {"faa", "fee"})
def test_find_stems(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
self.assertEqual(lexicon.find_stems("FOO", "barista"), {"foo"})
def load_lexicon(lexicon_file, pre_processor=lambda x: x):
lexicon = Lexicon()
partnum_to_key_regex = {
"1-": "P",
"1-A": "PA",
"1-M": "PM",
"1+": "I",
"2-": "F[AM]",
"2-A": "FA",
"2-M": "FM",
"3-": "A[AM][NPDSO]",
"3+": "A[AM]I",
"3+A": "AAI",
"3+M": "AMI",
"4-": "XA",
"4+": "YA",
"5-": "X[MP]",
"5+": "Y[MP]",
"6-": "AP[NPDSO]",
"6+": "API",
"7-": "FP",
"8-": "Z[MP]",
"M": "..M",
"F": "..F",
"N": "..N",
}
form_override = {}
accent_override = defaultdict(list)
with open(lexicon_file) as f:
for lemma, entry in yaml.load(f).items():
if "stems" in entry:
stems = []
for partnum, stems in sorted(
(entry["stems"] if entry.get("stems") else {}).items()):
key_regex = partnum_to_key_regex[partnum]
for stem, tag in split_stem_tags(stems):
lexicon.add(lemma, key_regex, pre_processor(stem), tag)
for key_regex, stems in entry.get("stem_overrides", []):
if stems is None:
continue
for stem, tag in split_stem_tags(stems):
lexicon.add(lemma, key_regex, pre_processor(stem), tag)
for key, form in entry.get("forms", {}).items():
form_override[(lemma, key)] = form
for key_regex, form in entry.get("accents", []):
accent_override[lemma].append((key_regex, form))
return lexicon, form_override, accent_override
def test_find_stems(self):
lexicon = Lexicon()
lexicon.add("FOO", "bar", "foo")
self.assertEqual(lexicon.find_stems("FOO", "barista"), {"foo"})