def test_regex_lemmatizer(self):
"""Test regex_lemmatizer()"""
sub = [("(.)ab(o|is|it|imus|itis|unt)$", r"\1o")]
lemmatizer = RegexpLemmatizer(sub)
test_str = "amabimus"
target = [("amabimus", "amo")]
tokenizer = LatinWordTokenizer()
test_str = test_str.lower()
test_str = replace_jv(test_str)
tokens = tokenizer.tokenize(test_str)
lemmas = lemmatizer.lemmatize(tokens)
self.assertEqual(lemmas, target)
def test_regex_lemmatizer(self):
"""Test regex_lemmatizer()"""
sub = [('(.)ab(o|is|it|imus|itis|unt)$', r'\1o')]
lemmatizer = RegexpLemmatizer(sub)
test_str = 'amabimus'
target = [('amabimus', 'amo')]
jv_replacer = JVReplacer()
tokenizer = WordTokenizer('latin')
test_str = test_str.lower()
test_str = jv_replacer.replace(test_str)
tokens = tokenizer.tokenize(test_str)
lemmas = lemmatizer.lemmatize(tokens)
self.assertEqual(lemmas, target)
def __init__(self: object, default: str = None, backoff: object = None):
"""
RomanNumeralLemmatizer
:type default: str
:param default: Default replacement for lemma; 'NUM' in given pattern
"""
regexps = [
(r'(?=^[MDCLXVUI]+$)(?=^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|IU|V?I{0,3}|U?I{0,3})$)', 'NUM'),
(r'(?=^[mdclxvui]+$)(?=^m{0,4}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|iu|v?i{0,3}|u?i{0,3})$)', 'NUM')
]
RegexpLemmatizer.__init__(self, regexps, backoff)
self._regexs = [(re.compile(regexp), pattern,) for regexp, pattern in regexps]
self.default = default
def test_regex_lemmatizer(self):
"""Test regex_lemmatizer()"""
sub = [('(.)ab(o|is|it|imus|itis|unt)$', r'\1o')]
lemmatizer = RegexpLemmatizer(sub)
test_str = 'amabimus'
target = [('amabimus', 'amo')]
jv_replacer = JVReplacer()
tokenizer = WordTokenizer('latin')
test_str = test_str.lower()
test_str = jv_replacer.replace(test_str)
tokens = tokenizer.tokenize(test_str)
lemmas = lemmatizer.lemmatize(tokens)
self.assertEqual(lemmas, target)
def _define_lemmatizer(self: object):
# Suggested backoff chain--should be tested for optimal order
self.backoff0 = None
self.backoff1 = IdentityLemmatizer(verbose=self.VERBOSE)
self.backoff2 = DictLemmatizer(
lemmas=self.GREEK_OLD_MODEL,
source="Morpheus Lemmas",
backoff=self.backoff1,
verbose=self.VERBOSE,
)
self.backoff3 = RegexpLemmatizer(
self.greek_sub_patterns,
source="CLTK Greek Regex Patterns",
backoff=self.backoff2,
verbose=self.VERBOSE,
)
self.backoff4 = UnigramLemmatizer(
self.train_sents,
source="CLTK Sentence Training Data",
backoff=self.backoff3,
verbose=self.VERBOSE,
)
self.backoff5 = DictLemmatizer(
lemmas=self.GREEK_MODEL,
source="Greek Model",
backoff=self.backoff4,
verbose=self.VERBOSE,
)
self.lemmatizer = self.backoff5
def _define_lemmatizer(self: object):
# Suggested backoff chain--should be tested for optimal order
self.backoff0 = None
self.backoff1 = IdentityLemmatizer(verbose=self.VERBOSE)
self.backoff2 = DictLemmatizer(lemmas=self.LATIN_OLD_MODEL, source='Morpheus Lemmas', backoff=self.backoff1, verbose=self.VERBOSE)
self.backoff3 = RegexpLemmatizer(self.latin_sub_patterns, source='CLTK Latin Regex Patterns', backoff=self.backoff2, verbose=self.VERBOSE)
self.backoff4 = UnigramLemmatizer(self.train_sents, source='CLTK Sentence Training Data', backoff=self.backoff3, verbose=self.VERBOSE)
self.backoff5 = DictLemmatizer(lemmas=self.LATIN_MODEL, source='Latin Model', backoff=self.backoff4, verbose=self.VERBOSE)
self.lemmatizer = self.backoff5
def evaluate(self: object):
if self.VERBOSE:
raise AssertionError("evaluate() method only works when verbose: bool = False")
return self.lemmatizer.evaluate(self.test_sents)
def __repr__(self: object):
return f'<BackoffLatinLemmatizer v0.2>'
if __name__ == '__main__':
from pprint import pprint
l1 = DefaultLemmatizer('UNK', verbose=True)
l2 = DictLemmatizer(lemmas={'arma': 'arma', 'uirum': 'uir'}, backoff=l1, verbose=True)
l3 = UnigramLemmatizer(train=[[('cano', 'cano'), ('.', 'punc')],], backoff=l2, verbose=True)
l4 = RegexpLemmatizer(regexps=[('(.)tat(is|i|em|e|es|um|ibus)$', r'\1tas'),], backoff=l3, verbose=True)
lemmas = l4.lemmatize('arma uirum -que cano nobilitatis .'.split())
pprint(lemmas)
# [('arma', 'arma', <UnigramLemmatizer: [[('res', 'res'), ...], ...]>),
# ('uirum', 'uir', <UnigramLemmatizer: [[('res', 'res'), ...], ...]>),
# ('-que', '-que', <DictLemmatizer: {'!': 'punc', ...}>),
# ('cano', 'cano', <DictLemmatizer: {'-nam': 'nam', ...}>),
# ('nobilitatis',
# 'nobilitas',
# <RegexpLemmatizer: [('(bil)(is|i|e...es|ium|ibus)$', '\\1is'), ...]>),
# ('.', 'punc', <DictLemmatizer: {'!': 'punc', ...}>)]
print('\n')
bll = BackoffLatinLemmatizer(seed=5, verbose=False)