def wordless_pos_tag(main,
tokens,
lang,
pos_tagger='default',
tagset='custom'):
tokens_tagged = []
# Check if the first token is empty
if tokens and tokens[0] == '':
first_token_empty = True
else:
first_token_empty = False
tokens = [str(token) for token in tokens if token]
if pos_tagger == 'default':
pos_tagger = main.settings_custom['pos_tagging']['pos_taggers'][lang]
wordless_text_utils.check_pos_taggers(main,
lang=lang,
pos_tagger=pos_tagger)
# Chinese
if pos_tagger == main.tr('jieba - Chinese POS Tagger'):
tokens_tagged = jieba.posseg.cut(' '.join(tokens))
# Dutch, English, French, German, Greek (Modern), Italian, Portuguese, Spanish
elif 'spaCy' in pos_tagger:
nlp = main.__dict__[f'spacy_nlp_{lang}']
doc = spacy.tokens.Doc(nlp.vocab, words=tokens)
nlp.tagger(doc)
tokens_tagged = [(token.text, token.tag_) for token in doc]
# English & Russian
elif pos_tagger == main.tr('NLTK - Perceptron POS Tagger'):
tokens_tagged = nltk.pos_tag(tokens, lang=lang)
# Japanese
elif pos_tagger == main.tr('nagisa - Japanese POS Tagger'):
import nagisa
tokens_tagged = zip(tokens, nagisa.postagging(tokens))
# Russian & Ukrainian
elif pos_tagger == main.tr('pymorphy2 - Morphological Analyzer'):
if lang == 'rus':
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='ru')
elif lang == 'ukr':
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='uk')
for token in tokens:
tokens_tagged.append(
(token, morphological_analyzer.parse(token)[0].tag._POS))
# Thai
elif pos_tagger == main.tr(
'PyThaiNLP - Perceptron POS Tagger - ORCHID Corpus'):
tokens_tagged = pythainlp.tag.pos_tag(tokens,
engine='perceptron',
corpus='orchid')
elif pos_tagger == main.tr(
'PyThaiNLP - Perceptron POS Tagger - PUD Corpus'):
tokens_tagged = pythainlp.tag.pos_tag(tokens,
engine='perceptron',
corpus='pud')
# Tibetan
elif pos_tagger == main.tr('botok - Tibetan POS Tagger'):
word_tokenizer = main.settings_custom['word_tokenization'][
'word_tokenizers'][lang]
botok_tokenizer = wordless_text_utils.check_botok_tokenizers(
main, word_tokenizer=word_tokenizer)
tokens = botok_tokenizer.tokenize(' '.join(tokens))
for token in tokens:
if token.pos:
tokens_tagged.append((token.text, token.pos))
else:
tokens_tagged.append((token.text, token.chunk_type))
# Vietnamese
elif pos_tagger == main.tr('Underthesea - Vietnamese POS Tagger'):
tokens_tagged = underthesea.pos_tag(' '.join(tokens))
# Convert to Universal Tagset
if (tagset == 'custom'
and main.settings_custom['pos_tagging']['to_universal_pos_tags']
or tagset == 'universal'):
mappings = {
tag: tag_universal
for tag, tag_universal, _, _ in main.settings_custom['tagsets']
['mappings'][lang][pos_tagger]
}
tokens_tagged = list(tokens_tagged)
# Issue warnings if any tag is missing from the mapping table
for _, tag in tokens_tagged:
if tag not in mappings:
print(
f'Warning: tag "{tag}" is missing from the {wordless_conversion.to_lang_text(main, lang)} mapping table!'
)
tokens_tagged = [(token, mappings.get(tag, 'X'))
for token, tag in tokens_tagged]
# Strip empty tokens and strip whitespace in tokens
tokens_tagged = [(token.strip(), tag) for token, tag in tokens_tagged
if token.strip()]
# Add the first empty token (if any)
if first_token_empty:
tokens_tagged.insert(0, ('', ''))
return tokens_tagged
def wordless_lemmatize(main,
tokens,
lang,
text_type=('untokenized', 'untagged'),
lemmatizer='default'):
empty_offsets = []
mapping_lemmas = {}
lemmas = []
tokens = [str(token) for token in tokens]
re_tags_all = wordless_matching.get_re_tags(main, tags='all')
re_tags_pos = wordless_matching.get_re_tags(main, tags='pos')
re_tags_non_pos = wordless_matching.get_re_tags(main, tags='non_pos')
if text_type[1] == 'tagged_both':
tags = [''.join(re.findall(re_tags_all, token)) for token in tokens]
tokens = [re.sub(re_tags_all, '', token) for token in tokens]
elif text_type[1] == 'tagged_pos':
tags = [''.join(re.findall(re_tags_pos, token)) for token in tokens]
tokens = [re.sub(re_tags_pos, '', token) for token in tokens]
elif text_type[1] == 'tagged_non_pos':
tags = [
''.join(re.findall(re_tags_non_pos, token)) for token in tokens
]
tokens = [re.sub(re_tags_non_pos, '', token) for token in tokens]
else:
tags = [''] * len(tokens)
# Record empty tokens
for i, token in reversed(list(enumerate(tokens))):
if not token.strip():
empty_offsets.append(i)
tokens.remove(token)
wordless_text_utils.check_lemmatizers(main, lang)
if tokens and lang in main.settings_global['lemmatizers']:
if lemmatizer == 'default':
lemmatizer = main.settings_custom['lemmatization']['lemmatizers'][
lang]
# Dutch, English, French, German, Greek (Modern), Italian, Portuguese, Spanish
if 'spaCy' in lemmatizer:
nlp = main.__dict__[f'spacy_nlp_{lang}']
doc = spacy.tokens.Doc(nlp.vocab, words=tokens)
nlp.tagger(doc)
lemmas = [token.lemma_ for token in doc]
# English
elif lemmatizer == main.tr('NLTK - WordNet Lemmatizer'):
word_net_lemmatizer = nltk.WordNetLemmatizer()
for token, pos in wordless_pos_tag(
main,
tokens,
lang='eng',
pos_tagger='NLTK - Perceptron POS Tagger',
tagset='universal'):
if pos == 'ADJ':
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.ADJ))
elif pos in ['NOUN', 'PROPN']:
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.NOUN))
elif pos == 'ADV':
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.ADV))
elif pos in ['VERB', 'AUX']:
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.VERB))
else:
lemmas.append(word_net_lemmatizer.lemmatize(token))
# Greek (Ancient)
elif lemmatizer == main.tr(
'lemmalist-greek - Greek (Ancient) Lemma List'):
with open(wordless_misc.get_normalized_path(
'lemmatization/lemmalist-greek/lemmalist-greek.txt'),
'r',
encoding='utf_8') as f:
for line in f.readlines():
line = line.rstrip()
if line:
lemma, *words = line.split()
for word in words:
mapping_lemmas[word] = lemma
# Russian & Ukrainian
elif lemmatizer == main.tr('pymorphy2 - Morphological Analyzer'):
if lang == 'rus':
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='ru')
else:
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='uk')
for token in tokens:
lemmas.append(
morphological_analyzer.parse(token)[0].normal_form)
# Tibetan
elif lemmatizer == main.tr('botok - Tibetan Lemmatizer'):
word_tokenizer = main.settings_custom['word_tokenization'][
'word_tokenizers'][lang]
botok_tokenizer = wordless_text_utils.check_botok_tokenizers(
main, word_tokenizer=word_tokenizer)
tokens = botok_tokenizer.tokenize(' '.join(tokens))
for token in tokens:
if token.lemma:
lemmas.append(token.lemma)
else:
lemmas.append(token.text)
# Other Languages
elif 'Lemmatization Lists' in lemmatizer:
lang = wordless_conversion.to_iso_639_1(main, lang)
with open(wordless_misc.get_normalized_path(
f'lemmatization/Lemmatization Lists/lemmatization-{lang}.txt'
),
'r',
encoding='utf_8_sig') as f:
for line in f:
try:
lemma, word = line.rstrip().split('\t')
mapping_lemmas[word] = lemma
except:
pass
else:
lemmas = tokens
if mapping_lemmas:
lemmas = [mapping_lemmas.get(token, token) for token in tokens]
# Insert empty lemmas
for empty_offset in sorted(empty_offsets):
lemmas.insert(empty_offset, '')
return [lemma + tag for lemma, tag in zip(lemmas, tags)]
def wordless_word_tokenize(main,
text,
lang,
word_tokenizer='default',
flat_tokens=True):
tokens_hierarchical = []
if lang not in main.settings_global['word_tokenizers']:
lang = 'other'
if word_tokenizer == 'default':
word_tokenizer = main.settings_custom['word_tokenization'][
'word_tokenizers'][lang]
# Check initialization status of word (and sentence) tokenizers
if flat_tokens:
wordless_text_utils.check_word_tokenizers(
main, lang=lang, word_tokenizer=word_tokenizer)
else:
wordless_text_utils.check_tokenizers(main,
lang=lang,
word_tokenizer=word_tokenizer)
# NLTK
if 'NLTK' in word_tokenizer:
sentences = wordless_sentence_tokenize(main, text, lang)
if word_tokenizer == main.tr('NLTK - Penn Treebank Tokenizer'):
treebank_tokenizer = nltk.TreebankWordTokenizer()
for sentence in sentences:
tokens_hierarchical.append(
treebank_tokenizer.tokenize(sentence))
elif word_tokenizer == main.tr('NLTK - Twitter Tokenizer'):
tweet_tokenizer = nltk.TweetTokenizer()
for sentence in sentences:
tokens_hierarchical.append(tweet_tokenizer.tokenize(sentence))
elif word_tokenizer == main.tr('NLTK - NIST Tokenizer'):
nist_tokenizer = nltk.tokenize.nist.NISTTokenizer()
for sentence in sentences:
tokens_hierarchical.append(nist_tokenizer.tokenize(sentence))
elif word_tokenizer == main.tr('NLTK - Tok-tok Tokenizer'):
toktok_tokenizer = nltk.ToktokTokenizer()
for sentence in sentences:
tokens_hierarchical.append(toktok_tokenizer.tokenize(sentence))
# Sacremoses
elif 'Sacremoses' in word_tokenizer:
if flat_tokens:
sentences = [text]
else:
sentences = wordless_sentence_tokenize(main, text, lang)
if word_tokenizer == main.tr('Sacremoses - Moses Tokenizer'):
moses_tokenizer = sacremoses.MosesTokenizer(
lang=wordless_conversion.to_iso_639_1(main, lang))
for sentence in sentences:
tokens_hierarchical.append(
moses_tokenizer.tokenize(sentence, escape=False))
elif word_tokenizer == main.tr('Sacremoses - Penn Treebank Tokenizer'):
moses_tokenizer = sacremoses.MosesTokenizer(
lang=wordless_conversion.to_iso_639_1(main, lang))
for sentence in sentences:
tokens_hierarchical.append(
moses_tokenizer.penn_tokenize(sentence))
# spaCy
elif 'spaCy' in word_tokenizer:
nlp = main.__dict__[f'spacy_nlp_{lang}']
doc = nlp(text)
# See Issue #3479: https://github.com/explosion/spaCy/issues/3479
doc.is_parsed = True
if flat_tokens:
tokens_hierarchical.append([token.text for token in doc])
else:
for sentence in doc.sents:
tokens_hierarchical.append(
[token.text for token in sentence.as_doc()])
# syntok
elif word_tokenizer == 'syntok - Word Tokenizer':
syntok_tokenizer = syntok.tokenizer.Tokenizer()
if flat_tokens:
tokens_hierarchical.append(
[token.value for token in syntok_tokenizer.tokenize(text)])
else:
for para in syntok.segmenter.analyze(text):
for sentence in para:
tokens_hierarchical.append(
[token.value for token in sentence])
# Chinese & Japanese
elif ('jieba' in word_tokenizer or 'nagisa' in word_tokenizer
or 'Wordless' in word_tokenizer):
if flat_tokens:
sentences = [text]
else:
sentences = wordless_sentence_tokenize(main, text, lang=lang)
# Chinese
if word_tokenizer == main.tr('jieba - Chinese Word Tokenizer'):
for sentence in sentences:
tokens_hierarchical.append(jieba.cut(sentence))
elif word_tokenizer == main.tr(
'Wordless - Chinese Character Tokenizer'):
for sentence in sentences:
tokens = []
non_han_start = 0
for i, char in enumerate(sentence):
if i >= non_han_start:
if wordless_checking_unicode.is_han(char):
tokens.append(char)
non_han_start += 1
else:
# English
if wordless_checking_unicode.is_eng(char):
for j, char in enumerate(sentence[i:]):
if i + j + 1 == len(
sentence
) or not wordless_checking_unicode.is_eng(
sentence[i + j + 1]):
tokens.extend(
wordless_word_tokenize(
main,
sentence[non_han_start:i + j +
1],
lang='eng'))
non_han_start = i + j + 1
break
# Other Languages
else:
for j, char in enumerate(sentence[i:]):
if i + j + 1 == len(
sentence
) or wordless_checking_unicode.is_han(
sentence[i + j + 1]):
tokens.extend(
wordless_word_tokenize(
main,
sentence[non_han_start:i + j +
1],
lang='other'))
non_han_start = i + j + 1
break
tokens_hierarchical.append(tokens)
# Japanese
elif word_tokenizer == main.tr('nagisa - Japanese Word Tokenizer'):
import nagisa
for sentence in sentences:
tokens_hierarchical.append(nagisa.tagging(str(sentence)).words)
elif word_tokenizer == main.tr('Wordless - Japanese Kanji Tokenizer'):
for sentence in sentences:
tokens = []
non_han_start = 0
for i, char in enumerate(sentence):
if i >= non_han_start:
if wordless_checking_unicode.is_han(char):
tokens.append(char)
non_han_start += 1
else:
# Japanese Kana
if wordless_checking_unicode.is_kana(char):
for j, char in enumerate(sentence[i:]):
if i + j + 1 == len(
sentence
) or not wordless_checking_unicode.is_kana(
sentence[i + j + 1]):
tokens.extend(
wordless_word_tokenize(
main,
sentence[non_han_start:i + j +
1],
lang='jpn'))
non_han_start = i + j + 1
break
# English
elif wordless_checking_unicode.is_eng(char):
for j, char in enumerate(sentence[i:]):
if i + j + 1 == len(
sentence
) or not wordless_checking_unicode.is_eng(
sentence[i + j + 1]):
tokens.extend(
wordless_word_tokenize(
main,
sentence[non_han_start:i + j +
1],
lang='eng'))
non_han_start = i + j + 1
break
# Other Languages
else:
for j, char in enumerate(sentence[i:]):
if i + j + 1 == len(
sentence
) or wordless_checking_unicode.is_han(
sentence[i + j + 1]):
tokens.extend(
wordless_word_tokenize(
main,
sentence[non_han_start:i + j +
1],
lang='other'))
non_han_start = i + j + 1
break
tokens_hierarchical.append(tokens)
# Russian
elif word_tokenizer == 'razdel - Russian Word Tokenizer':
if flat_tokens:
sentences = [text]
else:
sentences = wordless_sentence_tokenize(main, text, lang='rus')
for sentence in sentences:
tokens_hierarchical.append(
[token.text for token in razdel.tokenize(sentence)])
# Thai
elif 'PyThaiNLP' in word_tokenizer:
# Preserve sentence boundaries
sentences = wordless_sentence_tokenize(
main,
text,
lang='tha',
sentence_tokenizer='PyThaiNLP - Thai Sentence Tokenizer')
if word_tokenizer == main.tr(
'PyThaiNLP - Maximum Matching Algorithm + TCC'):
for sentence in sentences:
tokens_hierarchical.append(
pythainlp.tokenize.word_tokenize(sentence, engine='newmm'))
elif word_tokenizer == main.tr(
'PyThaiNLP - Maximum Matching Algorithm'):
for sentence in sentences:
tokens_hierarchical.append(
pythainlp.tokenize.word_tokenize(sentence, engine='mm'))
elif word_tokenizer == main.tr('PyThaiNLP - Longest Matching'):
for sentence in sentences:
tokens_hierarchical.append(
pythainlp.tokenize.word_tokenize(
sentence, engine='longest-matching'))
# Tibetan
elif 'botok' in word_tokenizer:
if flat_tokens:
sentences = [text]
else:
sentences = wordless_sentence_tokenize(main, text, lang='bod')
botok_tokenizer = wordless_text_utils.check_botok_tokenizers(
main, word_tokenizer)
for sentence in sentences:
tokens_hierarchical.append(
[token.text for token in botok_tokenizer.tokenize(sentence)])
# Vietnamese
elif word_tokenizer == main.tr('Underthesea - Vietnamese Word Tokenizer'):
if flat_tokens:
sentences = [text]
else:
sentences = wordless_sentence_tokenize(
main,
text,
lang='vie',
sentence_tokenizer='Underthesea - Vietnamese Sentence Tokenizer'
)
for sentence in sentences:
tokens_hierarchical.append(underthesea.word_tokenize(
str(sentence)))
# Remove empty tokens and strip whitespace
for i, sentence in enumerate(tokens_hierarchical):
tokens_hierarchical[i] = [
token.strip() for token in sentence if token.strip()
]
# Record token boundaries
if lang in ['zho_cn', 'zho_tw', 'jpn']:
for sentence in tokens_hierarchical:
if sentence:
sentence[-1] = wordless_text.Wordless_Token(
sentence[-1], boundary='', sentence_ending=True)
else:
for sentence in tokens_hierarchical:
if sentence:
sentence[-1] = wordless_text.Wordless_Token(
sentence[-1], boundary=' ', sentence_ending=True)
# Clause tokenization
if not flat_tokens:
for i, sentence in enumerate(tokens_hierarchical):
tokens_hierarchical[i] = wordless_clause_tokenize(
main, sentence, lang)
# Flatten tokens
tokens_flat = list(wordless_misc.flatten_list(tokens_hierarchical))
if flat_tokens:
return tokens_flat
else:
return tokens_hierarchical