def lemm_doc(doc, language='de'):
'''
Lemms all words in doc.
'''
doc = list(filter(lambda x: x != '', doc))
vocabular = file_as_dict(file_map[language])
return list(map(lambda x: lemm(x, vocabular), doc))
def lemm_doc_stream(doc, language):
'''
Lemms all words in stdin stream.
'''
vocabular = file_as_dict(file_map[language])
doc = (x.replace('\n', '') for x in doc)
doc = (x for x in doc if x != '')
return map(lambda x: lemm(x, vocabular), doc)
def category_to_answer(map, file):
map = file_as_dict(map, sep=';', local=False)
map = {k.lower(): v for k, v in map.items()}
cat = file_as_list(file, local=False)
for c in cat:
if c not in map:
print(c)
def test_file_as_dict(tmp_path):
words_test = tmp_path / 'test.csv'
words_test.write_text(CONTENT)
words = file_as_dict(words_test, sep=';', local=False)
assert words == {
'category1': 'answer1',
'category2': 'answer2',
'category3': 'answer3'
}
def ionesoft_lemming(question):
vocabular = file_as_dict('res/custom_ch_vocabular.txt')
question = ionesoft_normalize(question)
return lemm(question, vocabular)
def ionesoft_normalize(question):
grammar = file_as_dict('res/custom_ch_grammar.txt')
normalized = normalize(question)
return correct(normalized, grammar)
def stackexchange_lemming(question):
vocabular = file_as_dict('res/custom_en_vocabular.txt')
question = stackexchange_normalize(question)
return lemm(question, vocabular)
def correct_grammar_stream(doc, grammar='res/custom_ch_grammar.txt'):
doc = (x for x in doc if x != '\n')
grammar = file_as_dict(grammar)
return map(lambda x: correct(x, grammar), doc)
def correct_grammar(doc, grammar='res/custom_ch_grammar.txt'):
grammar = file_as_dict(grammar)
doc = list(filter(lambda x: x != '', doc))
return list(map(lambda x: correct(x, grammar), doc))