def testing_lemmatize(lang, lemmatizer):
lang_text = wordless_conversion.to_lang_text(main, lang)
print(f'{lang_text} / {lemmatizer}:')
wordless_text_utils.check_lemmatizers(main, lang, lemmatizer = lemmatizer)
lemmas = wordless_text_processing.wordless_lemmatize(main, globals()[f'tokens_{lang}'],
lang = lang,
lemmatizer = lemmatizer)
print(f"\t{lemmas}")
def testing_lemmatize(lang, lemmatizer):
lang_text = wordless_conversion.to_lang_text(main, lang)
print(f'{lang_text} / {lemmatizer}:')
tokens_sentences = wordless_text_processing.wordless_word_tokenize(
main, globals()[f'SENTENCE_{lang.upper()}'], lang=lang)
tokens = [token for tokens in tokens_sentences for token in tokens]
lemmas = wordless_text_processing.wordless_lemmatize(main,
tokens,
lang=lang,
lemmatizer=lemmatizer)
print(f"\t{lemmas}")
def test_lemmatize(lang, lemmatizer, show_results=False):
lang_text = wordless_conversion.to_lang_text(main, lang)
tokens = wordless_text_processing.wordless_word_tokenize(
main,
text=getattr(wordless_test_lang_examples, f'SENTENCE_{lang.upper()}'),
lang=lang)
lemmas = wordless_text_processing.wordless_lemmatize(main,
tokens=tokens,
lang=lang,
lemmatizer=lemmatizer)
if show_results:
print(lemmas)
if lang == 'ast':
assert lemmas == [
"L'asturianu", 'ser', 'unu', 'llingua', 'romance', 'propiu',
"d'Asturies,[1", ']', 'perteneciente', 'al', 'subgrupu',
'asturllionés', '.'
]
elif lang == 'bul':
assert lemmas == [
'Бъ̀лгарският', 'езѝк', 'съм', 'индоевропейски', 'език', 'от',
'група', 'на', 'южнославянските', 'език', '.'
]
elif lang == 'cat':
assert lemmas == [
'El', 'català', '(', 'denominació', 'oficial', 'a', 'Catalunya',
',', 'a', 'ell', 'Illes', 'Balears', ',', 'a', 'Andorra', ',', 'a',
'ell', 'ciutat', 'de', 'ell', 'Alguer', 'i', 'tradicional', 'a',
'Catalunya', 'Nord', ')', 'o', 'valencià', '(', 'denominació',
'oficial', 'al', 'País', 'Valencià', 'i', 'tradicional', 'al',
'Carxe', ')', 'ser', 'un', 'llengua', 'romànic', 'parlar', 'a',
'Catalunya', ',', 'ell', 'País', 'Valencià', '(', 'treure', 'de',
'algun', 'comarca', 'i', 'localitat', 'de', 'ell', 'interior', ')',
',', 'ell', 'Illes', 'Balears', ',', 'Andorra', ',', 'ell',
'Franja', 'de', 'Ponent', '(', 'a', 'ell', 'Aragó', ')', ',',
'ell', 'ciutat', 'de', 'ell', 'Alguer', '(', 'a', 'ell', 'illa',
'de', 'Sardenya', ')', ',', 'ell', 'Catalunya', 'del', 'Nord,[8',
']', 'ell', 'Carxe', '(', 'un', 'petit', 'territori', 'de',
'Múrcia', 'poblar', 'per', 'immigrar', 'valencians),[9][10', ']',
'i', 'en', 'petita', 'comunitat', 'arreu', 'del', 'món', '(',
'entrar', 'ell', 'qual', 'destacar', 'ell', 'de', 'ell',
'Argentina', ',', 'amb', '195.000', 'parlants).[11', ']'
]
elif lang == 'ces':
assert lemmas == [
'Čeština', 'neboli', 'český', 'jazyk', 'on', 'západoslovanský',
'jazyk', ',', 'blízký', 'slovenštině', ',', 'poté', 'lužické',
'srbštině', 'a', 'polštině', '.'
]
elif lang == 'nld':
assert lemmas == [
'het', 'nederlands', 'zijn', 'een', 'west-germaans', 'taal', 'en',
'de', 'moedertaal', 'van', 'de', 'veel', 'inwoner', 'van',
'nederland', ',', 'belgië', 'en', 'suriname', '.'
]
elif lang == 'eng':
if lemmatizer == 'Lemmatization Lists - English Lemma List':
assert lemmas == [
'English', 'be', 'a', 'West', 'Germanic', 'language', 'that',
'be', '1', 'speak', 'in', 'early', 'medieval', 'England',
'and', 'eventually', 'become', 'a', 'global', 'lingua',
'franca.[4][5', ']'
]
elif lemmatizer == 'NLTK - WordNet Lemmatizer':
assert lemmas == [
'English', 'be', 'a', 'West', 'Germanic', 'language', 'that',
'be', 'first', 'speak', 'in', 'early', 'medieval', 'England',
'and', 'eventually', 'become', 'a', 'global', 'lingua',
'franca.[4][5', ']'
]
elif lemmatizer == 'spaCy - English Lemmatizer':
assert lemmas == [
'English', 'be', 'a', 'West', 'germanic', 'language', 'that',
'be', 'first', 'speak', 'in', 'early', 'medieval', 'England',
'and', 'eventually', 'become', 'a', 'global', 'lingua',
'franca.[4][5', ']'
]
elif lang == 'est':
assert lemmas == [
'Eesti', 'kee', '(', 'varasem', 'nimetu', ':', 'maakeel', ')',
'olema', 'läänemeresoome', 'lõunarühma', 'kuuluma', 'kee', '.'
]
elif lang == 'fra':
if lemmatizer == 'Lemmatization Lists - French Lemma List':
assert lemmas == [
'Le', 'français', 'être', 'un', 'langue', 'indo-européen',
'de', 'le', 'famille', 'un', 'langue', 'roman', '.'
]
elif lemmatizer == 'spaCy - French Lemmatizer':
assert lemmas == [
'le', 'français', 'être', 'un', 'langue', 'indo-européen',
'de', 'le', 'famille', 'un', 'langue', 'roman', '.'
]
elif lang == 'glg':
assert lemmas == [
'O', 'galego', '(', '[', 'ɡaˈleɣo̝', ']', ')', 'ser', 'un',
'lingua', 'indoeuropeo', 'que', 'pertencer', 'á', 'póla', 'de',
'lingua', 'románico', '.'
]
elif lang == 'deu':
if lemmatizer == 'Lemmatization Lists - German Lemma List':
assert lemmas == [
'Die', 'deutsch', 'Sprache', 'bzw.', 'Deutsch', '(', '[',
'dɔʏ̯t͡ʃ', ']', ';', 'abkürzen', 'dt', '.', 'oder', 'dtsch',
'.', ')', 'sein', 'einen', 'westgermanische', 'Sprache', '.'
]
elif lemmatizer == 'spaCy - German Lemmatizer':
assert lemmas == [
'der', 'deutsch', 'Sprache', 'bzw.', 'Deutsch', '(', '[',
'dɔʏ̯t͡ʃ', ']', ';', 'abkürzen', 'dt', '.', 'oder', 'dtsch',
'.', ')', 'sein', 'einen', 'westgermanische', 'Sprache', '.'
]
elif lang == 'grc':
assert lemmas == [
'Με', 'τον', 'όρο', 'αρχαία', 'ελληνική', 'γλώσσα', 'εννοείται',
'μια', 'μορφή', 'της', 'ελληνικής', 'γλώσσας', ',', 'πού',
'ομιλούνταν', 'κατά', 'τους', 'αρχαϊκούς', 'χρόνους', 'και', 'την',
'κλασική', 'αρχαιότητα', '.'
]
elif lang == 'ell':
assert lemmas == [
'η', 'ελληνικός', 'γλώσσα', 'ανήκω', 'στην', 'ινδοευρωπαϊκός',
'οικογένεια[9', ']', 'και', 'συγκεκριμένα', 'στον', 'ελληνικό',
'κλάδο', ',', 'μαζί', 'με', 'την', 'τσακωνικός', ',', 'ενώ',
'είναι', 'η', 'επίσημη', 'γλώσσα', 'της', 'Ελλάδος', 'και', 'της',
'Κύπρου', '.'
]
elif lang == 'hun':
assert lemmas == [
'A', 'magyar', 'nyelv', 'az', 'uráli', 'nyelvcsalád', 'tag', ',',
'a', 'finnugor', 'nyelv', 'köz', 'tartozó', 'ugor', 'nyelv',
'egyik', '.'
]
elif lang == 'gle':
assert lemmas == [
'Is', 'ceann', 'de', 'na', 'teangach', 'Ceilteacha', 'í', 'an',
'Ghaeilge', '(', 'nó', 'Gaeilge', 'na', 'hÉireann', 'mar', 'a',
'tabhair', 'ar', 'corruair', ')', ',', 'agus', 'ceann', 'den',
'trí', 'ceann', 'de', 'teangach', 'Ceilteacha', 'air', 'a',
'tabhair', 'na', 'teangach', 'Gaelacha', '(', '.i.', 'an',
'Ghaeilge', ',', 'Gaeilge', 'na', 'hAlban', 'agus', 'Gaeilge',
'Mhanann', ')', 'go', 'áirithe', '.'
]
elif lang == 'ita':
assert lemmas == [
"L'", 'italiano', '(', '[', 'itaˈljaːno][Nota', '1', ']',
'ascolta[?·info', ']', ')', 'essere', 'una', 'lingua', 'romanzo',
'parlato', 'principalmente', 'in', 'Italia', '.'
]
elif lang == 'lit':
assert lemmas == [
'lietuvė', 'kalbėti', '–', 'ižti', 'baltas', 'prokalbės', 'kilęs',
'lietuvė', 'tauta', 'kalbėti', ',', '-PRON-', 'Lietuvoje', 'būti',
'valstybinis', ',', 'o', 'Europos', 'sąjunga', '–', 'viena',
'ižti', 'oficialus', 'kalbus', '.'
]
elif lang == 'glv':
assert lemmas == [
'She', 'Gaelg', '(', 'graït', ':', '/gɪlg/', ')', 'çhengey',
'Gaelagh', 'Mannin', '.'
]
elif lang == 'nob':
assert lemmas == [
'bokmål', 'være', 'en', 'varietet', 'av', 'norsk', 'språk', '.'
]
elif lang == 'fas':
assert lemmas == [
'فارسی', 'یا', 'پارسی', 'یکی', 'از', 'زبان\u200cهای',
'هندواروپایی', 'در', 'شاخهٔ', 'زبان\u200cهای', 'ایرانی', 'جنوب',
'غربی', 'است', 'که', 'در', 'کشورهای', 'ایران', '،', 'افغانستان،[۳',
']', 'تاجیکستان[۴', ']', 'را', 'ازبکستان[۵', ']', 'به', 'آن',
'سخن', 'می\u200cگویند', '.'
]
elif lang == 'por':
assert lemmas == [
'A', 'língua', 'portuguesar', ',', 'também', 'designar',
'português', ',', 'ser', 'umar', 'língua', 'românico', 'flexivo',
'ocidental', 'originar', 'o', 'galego-português', 'falar', 'o',
'Reino', 'da', 'Galiza', 'e', 'o', 'norte', 'de', 'Portugal', '.'
]
elif lang == 'ron':
assert lemmas == [
'Limba', 'român', 'fi', 'vrea', 'limbă', 'indo', '-', 'european',
',', 'din', 'grup', 'italic', 'și', 'din', 'subgrupul', 'oriental',
'al', 'limbă', 'romanice', '.'
]
elif lang == 'rus':
assert lemmas == [
'ру́сский', 'язы́к', '(', '[', 'ˈruskʲɪi̯', 'jɪˈzɨk', ']',
'информация', 'о', 'файл', 'слушать', ')', '[', '~', '3', ']', '[',
'⇨', ']', '—', 'один', 'из', 'восточнославянский', 'язык', ',',
'национальный', 'язык', 'русский', 'народ', '.'
]
elif lang == 'gla':
assert lemmas == [
"'S", 'i', 'cànan', 'dùthchasach', 'na', 'h', '-', 'Alba', 'a',
'th', "'", 'anns', 'a', "'", 'Ghàidhlig', '.'
]
elif lang == 'slk':
assert lemmas == [
'Slovenčina', 'patriť', 'do', 'skupina', 'západoslovanský',
'jazyk', '(', 'spolu', 's', 'čeština', ',', 'poľština', ',',
'horný', 'as', 'dolný', 'lužickou', 'srbčina', 'as', 'kašubčinou',
')', '.'
]
elif lang == 'slv':
assert lemmas == [
'Slovenščina', '[', 'slovénščina', ']', '/', '[', 'sloˈʋenʃtʃina',
']', 'onbiti', 'združen', 'naziv', 'za', 'uraden', 'knjižen',
'jezik', 'Slovenec', 'in', 'skupen', 'ime', 'za', 'narečje', 'in',
'govoriti', ',', 'ki', 'on', 'govoriti', 'ali', 'biti', 'on',
'nekoč', 'govoriti', 'Slovenec', '.'
]
elif lang == 'spa':
assert lemmas == [
'El', 'español', 'o', 'castellano', 'ser', 'uno', 'lengua',
'romance', 'procedente', 'del', 'latín', 'hablar', '.'
]
elif lang == 'swe':
assert lemmas == [
'Svenska', '(', 'svensk', '(', 'info', ')', ')', 'vara', 'en',
'östnordiskt', 'språka', 'som', 'tala', 'av', 'ungefär', 'tio',
'miljon', 'person', 'främst', 'i', 'Sverige', 'där', 'språk',
'hare', 'man', 'dominant', 'ställning', 'som', 'huvudspråk', ',',
'mena', 'även', 'som', 'en', 'en', 'nationalspråk', 'i', 'Finland',
'och', 'som', 'enda', 'officiell', 'språka', 'på', 'Åland', '.'
]
elif lang == 'bod':
assert lemmas == [
'༄༅། ། ', 'རྒྱ་གར་', 'སྐད་', 'དུ་', ' ། ', 'བོ་', ' དྷི་', ' སཏྭ་',
' ཙརྻ་', 'ཨ་བ་', 'ཏ་', 'ར་', ' ། ', 'བོད་སྐད་', 'དུ་', ' ། ',
'བྱང་ཆུབ་', 'སེམས་དཔའ་', 'གི་', 'སྤྱོད་པ་', 'ལ་', 'འཇུག་པ་',
' ། ། ', 'སངས་རྒྱས་', 'དང་', 'བྱང་ཆུབ་', 'སེམས་དཔའ་', 'ཐམས་ཅད་',
'ལ་', 'ཕྱག་', 'འཚལ་', 'ལོ་', ' ། ། ', 'བདེ་གཤེགས་', 'ཆོ་', 'ཀྱི་',
'སྐུ་', 'མངའ་', 'སྲ་', 'བཅའ་', 'དང་', ' ། ། ', 'ཕྱག་འོས་', 'ཀུན་',
'ལ་', 'ཀྱང་', 'གུས་པ་', 'ལ་', 'ཕྱག་', 'འཚལ་', 'ཏེ་', ' ། ། ',
'བདེ་གཤེགས་', 'སྲ་', 'ཀྱི་', 'སྡོམ་', 'ལ་', 'འཇུག་པ་', 'ནི་',
' ། ། ', 'ལུང་', 'བཞིན་', 'མདོར་བསྡུས་', 'ན་', 'ནི་', 'བརྗོད་པ་',
'ལ་', 'བྱ་', ' ། །'
]
elif lang == 'ukr':
if lemmatizer == 'Lemmatization Lists - Ukrainian Lemma List':
assert lemmas == [
'Украї́нська', 'мо́ва', '(', 'МФА', ':', '[',
'ukrɑ̽ˈjɪnʲsʲkɑ̽', 'ˈmɔwɑ̽', ']', ',', 'історичний', 'назвати',
'—', 'ру́ська', ',', 'руси́нська[9][10][11', ']', '[', '*',
'2', ']', ')', '—', 'національний', 'мова', 'українець', '.'
]
elif lemmatizer == 'pymorphy2 - Morphological Analyzer':
assert lemmas == [
'украї́нський', 'мо́вий', '(', 'мфа', ':', '[',
'ukrɑ̽ˈjɪnʲsʲkɑ̽', 'ˈmɔwɑ̽', ']', ',', 'історичний', 'назва',
'—', 'ру́ський', ',', 'руси́нська[9][10][11', ']', '[', '*',
'2', ']', ')', '—', 'національний', 'мова', 'українець', '.'
]
elif lang == 'cym':
assert lemmas == [
'Aelod', "o'r", 'cangen', 'Frythonaidd', "o'r", 'iaith',
'Celtaidd', 'a', 'siarad', 'bod', 'brodorol', 'yn', 'Nghymru', ',',
'can', 'Gymry', 'a', 'pobl', 'arall', 'aredig', 'gwasgar', 'bod',
'Lloegr', ',', 'a', 'can', 'cymuno', 'bechan', 'bod', 'Y',
'Wladfa', ',', 'gwybod', 'Ariannin[7', ']', "yw'r", 'Gymraeg', '(',
'hefyd', 'Cymraeg', 'heb', 'yr', 'bannod', ')', '.'
]
def wordless_process_tokens(text, token_settings):
main = text.main
settings = copy.deepcopy(token_settings)
# Token Settings
if settings['use_tags']:
settings['ignore_tags'] = settings['ignore_tags_tags']
settings['ignore_tags_type'] = settings['ignore_tags_type_tags']
# Punctuations
if not settings['puncs']:
i_tokens = 0
# Mark tokens to be removed
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_punc(token):
clause[i] = ''
text.tags_pos[i_tokens + i] = ''
text.tags_non_pos[i_tokens + i] = ''
text.tags_all[i_tokens + i] = ''
i_tokens += len(clause)
# Remove punctuations
for para in text.tokens_hierarchical:
for sentence in para:
for i, clause in enumerate(sentence):
sentence[i] = [token for token in clause if token]
text.tags_pos = [tags for tags in text.tags_pos if tags != '']
text.tags_non_pos = [tags for tags in text.tags_pos if tags != '']
text.tags_all = [tags for tags in text.tags_pos if tags != '']
# Lemmatize all tokens
if not settings['use_tags'] and settings['lemmatize_tokens']:
for para in text.tokens_hierarchical:
for sentence in para:
for i, clause in enumerate(sentence):
sentence[i] = wordless_text_processing.wordless_lemmatize(
main, clause,
lang = text.lang
)
# Treat as all lowercase
if settings['treat_as_lowercase']:
for para in text.tokens_hierarchical:
for sentence in para:
for i, clause in enumerate(sentence):
sentence[i] = [token.lower() for token in clause]
text.tags_pos = [[tag.lower() for tag in tags] for tags in text.tags_pos]
text.tags_non_pos = [[tag.lower() for tag in tags] for tags in text.tags_non_pos]
text.tags_all = [[tag.lower() for tag in tags] for tags in text.tags_all]
# Words
if settings['words']:
# Lowercase
if not settings['lowercase']:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_word_lowercase(token):
clause[i] = ''
# Uppercase
if not settings['uppercase']:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_word_uppercase(token):
clause[i] = ''
# Title Case
if not settings['title_case']:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_word_title_case(token):
clause[i] = ''
else:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_word(token):
clause[i] = ''
# Numerals
if not settings['nums']:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
if wordless_checking_token.is_token_num(token):
clause[i] = ''
# Filter stop words
if settings['filter_stop_words']:
for para in text.tokens_hierarchical:
for sentence in para:
for i, clause in enumerate(sentence):
sentence[i] = wordless_text_processing.wordless_filter_stop_words(
main, clause,
lang = text.lang
)
# Ignore tags
i_token = 0
if settings['ignore_tags']:
# Ignore all tags
if settings['ignore_tags_type'] == main.tr('all'):
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = (token, [])
# Ignore POS tags
elif settings['ignore_tags_type'] == main.tr('POS'):
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = (token, text.tags_non_pos[i_token + i])
i_token += len(clause)
# Ignore non-POS tags
elif settings['ignore_tags_type'] == main.tr('non-POS'):
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = (token, text.tags_pos[i_token + i])
i_token += len(clause)
else:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = (token, text.tags_all[i_token + i])
i_token += len(clause)
# Use tags only
if settings['use_tags']:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = clause[i][1]
else:
for para in text.tokens_hierarchical:
for sentence in para:
for clause in sentence:
for i, token in enumerate(clause):
clause[i] = f"{clause[i][0]}{''.join(clause[i][1])}"
text.tokens_flat = list(wordless_misc.flatten_list(text.tokens_hierarchical))
return text
def wordless_process_tokens(text, token_settings):
main = text.main
tokens = text.tokens.copy()
settings = copy.deepcopy(token_settings)
# Token Settings
if settings['use_tags']:
settings['ignore_tags'] = settings['ignore_tags_tags']
settings['ignore_tags_type'] = settings['ignore_tags_type_tags']
# Punctuations
if not settings['puncs']:
for i, token in reversed(list(enumerate(tokens))):
if wordless_checking_token.is_token_punc(token):
del tokens[i]
del text.tags_pos[i]
del text.tags_non_pos[i]
del text.tags_all[i]
# Lemmatize all tokens
if not settings['use_tags'] and settings['lemmatize_tokens']:
tokens = wordless_text_processing.wordless_lemmatize(main,
tokens,
lang=text.lang)
# Treat as all lowercase
if settings['treat_as_lowercase']:
tokens = [token.lower() for token in tokens]
text.tags_pos = [[tag.lower() for tag in tags]
for tags in text.tags_pos]
text.tags_non_pos = [[tag.lower() for tag in tags]
for tags in text.tags_non_pos]
text.tags_all = [[tag.lower() for tag in tags]
for tags in text.tags_all]
text.tokens = copy.deepcopy(tokens)
# Words
if settings['words']:
# Lowercase
if not settings['lowercase']:
for i, token in enumerate(tokens):
if wordless_checking_token.is_token_word_lowercase(token):
tokens[i] = ''
# Uppercase
if not settings['uppercase']:
for i, token in enumerate(tokens):
if wordless_checking_token.is_token_word_uppercase(token):
tokens[i] = ''
# Title Case
if not settings['title_case']:
for i, token in enumerate(tokens):
if wordless_checking_token.is_token_word_title_case(token):
tokens[i] = ''
else:
for i, token in enumerate(tokens):
if wordless_checking_token.is_token_word(token):
tokens[i] = ''
# Numerals
if not settings['nums']:
for i, token in enumerate(tokens):
if wordless_checking_token.is_token_num(token):
tokens[i] = ''
# Filter stop words
if settings['filter_stop_words']:
tokens_filtered = wordless_text_processing.wordless_filter_stop_words(
main, [token for token in tokens], lang=text.lang)
for i, token in enumerate(tokens):
if token not in tokens_filtered:
tokens[i] = ''
# Ignore tags
if settings['ignore_tags']:
# Ignore all tags
if settings['ignore_tags_type'] == main.tr('all'):
tokens = [(token, []) for token in tokens]
text.tokens = [(token, []) for token in text.tokens]
# Ignore POS tags
elif settings['ignore_tags_type'] == main.tr('POS'):
tokens = [(token, tags)
for token, tags in zip(tokens, text.tags_non_pos)]
text.tokens = [
(token, tags)
for token, tags in zip(text.tokens, text.tags_non_pos)
]
# Ignore non-POS tags
elif settings['ignore_tags_type'] == main.tr('non-POS'):
tokens = [(token, tags)
for token, tags in zip(tokens, text.tags_pos)]
text.tokens = [(token, tags)
for token, tags in zip(text.tokens, text.tags_pos)]
else:
tokens = [(token, tags) for token, tags in zip(tokens, text.tags_all)]
text.tokens = [(token, tags)
for token, tags in zip(text.tokens, text.tags_all)]
return tokens
def match_ngrams(main, search_terms, tokens, lang, text_type, token_settings,
search_settings):
search_terms_matched = set()
settings = copy.deepcopy(search_settings)
re_tags_all = get_re_tags(main, tags='all')
re_tags_pos = get_re_tags(main, tags='pos')
re_tags_non_pos = get_re_tags(main, tags='non_pos')
search_term_tokens = [
search_term_token for search_term in search_terms
for search_term_token in search_term.split()
]
if search_settings['use_regex']:
regexes_matched = {
search_term_token: set()
for search_term_token in search_term_tokens
}
tokens_matched = {}
else:
tokens_matched = {
search_term_token: set()
for search_term_token in search_term_tokens
}
# Search Settings
if settings['match_tags']:
settings['match_inflected_forms'] = False
settings['ignore_tags'] = settings['ignore_tags_tags']
settings['ignore_tags_type'] = settings['ignore_tags_type_tags']
# Token Settings
if token_settings['use_tags']:
settings['match_inflected_forms'] = False
settings['match_tags'] = False
if token_settings['ignore_tags_tags']:
settings['ignore_tags'] = False
else:
if token_settings['ignore_tags']:
if token_settings['ignore_tags_type'] == main.tr('all'):
settings['ignore_tags'] = False
settings['match_tags'] = False
# Match Tags Only & Ignore Tags
if settings['match_tags']:
if settings['ignore_tags']:
if text_type[1] == 'untagged':
tokens_searched = []
else:
if settings['ignore_tags_type'] == main.tr('POS'):
if text_type[1] in ['tagged_both', 'tagged_non_pos']:
tokens_searched = [
''.join(re.findall(re_tags_non_pos, token))
for token in tokens
]
elif text_type[1] == 'tagged_pos':
tokens_searched = []
elif settings['ignore_tags_type'] == main.tr('non-POS'):
if text_type[1] in ['tagged_both', 'tagged_pos']:
tokens_searched = [
''.join(re.findall(re_tags_pos, token))
for token in tokens
]
elif text_type[1] == 'tagged_non_pos':
tokens_searched = []
else:
if text_type[1] == 'untagged':
tokens_searched = []
elif text_type[1] == 'tagged_pos':
tokens_searched = [
''.join(re.findall(re_tags_pos, token)) for token in tokens
]
elif text_type[1] == 'tagged_non_pos':
tokens_searched = [
''.join(re.findall(re_tags_non_pos, token))
for token in tokens
]
elif text_type[1] == 'tagged_both':
tokens_searched = [
''.join(re.findall(re_tags_all, token)) for token in tokens
]
else:
if settings['ignore_tags']:
if text_type[1] == 'untagged':
tokens_searched = tokens
else:
if settings['ignore_tags_type'] == main.tr('all'):
if text_type[1] == 'tagged_both':
tokens_searched = [
re.sub(re_tags_all, '', token) for token in tokens
]
elif text_type[1] == 'tagged_pos':
tokens_searched = [
re.sub(re_tags_pos, '', token) for token in tokens
]
elif text_type[1] == 'tagged_non_pos':
tokens_searched = [
re.sub(re_tags_non_pos, '', token)
for token in tokens
]
elif settings['ignore_tags_type'] == main.tr('POS'):
if text_type[1] in ['tagged_both', 'tagged_pos']:
tokens_searched = [
re.sub(re_tags_pos, '', token) for token in tokens
]
elif text_type[1] == 'tagged_non_pos':
tokens_searched = tokens
elif settings['ignore_tags_type'] == main.tr('non-POS'):
if text_type[1] in ['tagged_both', 'tagged_non_pos']:
tokens_searched = [
re.sub(re_tags_non_pos, '', token)
for token in tokens
]
elif text_type[1] == 'tagged_pos':
tokens_searched = tokens
else:
tokens_searched = tokens
if tokens_searched:
if settings['use_regex']:
for search_term_token in search_term_tokens:
if settings['match_whole_words']:
regex = fr'(^|\s+){search_term_token}(\s+|$)'
else:
regex = search_term_token
if settings['ignore_case']:
flags = re.IGNORECASE
else:
flags = 0
for token, token_searched in zip(tokens, tokens_searched):
if re.search(regex, token_searched, flags=flags):
regexes_matched[search_term_token].add(token)
tokens_matched[token] = set()
else:
for search_term_token in search_term_tokens:
regex = re.escape(search_term_token)
if settings['match_whole_words']:
regex = fr'(^|\s+){regex}(\s+|$)'
if settings['ignore_case']:
flags = re.IGNORECASE
else:
flags = 0
for token, token_searched in zip(tokens, tokens_searched):
if re.search(regex, token_searched, flags=flags):
tokens_matched[search_term_token].add(token)
if settings['match_inflected_forms']:
lemmas_searched = wordless_text_processing.wordless_lemmatize(
main, tokens_searched, lang, text_type)
lemmas_matched = wordless_text_processing.wordless_lemmatize(
main, list(tokens_matched), lang, text_type)
for token_matched, lemma_matched in zip(list(tokens_matched),
lemmas_matched):
lemma_matched = re.escape(lemma_matched)
lemma_matched = fr'(^|\s+){lemma_matched}(\s+|$)'
if settings['ignore_case']:
flags = re.IGNORECASE
else:
flags = 0
for token, lemma_searched in zip(tokens, lemmas_searched):
if re.search(lemma_matched, lemma_searched, flags=flags):
tokens_matched[token_matched].add(token)
if search_settings['use_regex']:
for search_term in search_terms:
search_term_tokens_matched = []
for search_term_token in search_term.split():
search_term_tokens_matched.append(set())
for regex_matched in regexes_matched[search_term_token]:
search_term_tokens_matched[-1].add(regex_matched)
search_term_tokens_matched[-1] |= set(
tokens_matched[regex_matched])
for item in itertools.product(*search_term_tokens_matched):
search_terms_matched.add(item)
else:
for search_term in search_terms:
search_term_tokens_matched = []
for search_term_token in search_term.split():
search_term_tokens_matched.append(
set(tokens_matched[search_term_token]))
for item in itertools.product(*search_term_tokens_matched):
search_terms_matched.add(item)
return search_terms_matched