def detect_text_type(main, file):
tagged_pos = False
tagged_non_pos = False
try:
with open(file['path'], 'r', encoding = file['encoding']) as f:
re_tags_pos = wordless_matching.get_re_tags(main, tags = 'pos')
re_tags_non_pos = wordless_matching.get_re_tags(main, tags = 'non_pos')
if main.settings_custom['auto_detection']['detection_settings']['number_lines_no_limit']:
for line in f:
if re.search(re_tags_pos, line):
tagged_pos = True
break
f.seek(0)
for line in f:
if re.search(re_tags_non_pos, line):
tagged_non_pos = True
break
else:
for i, line in enumerate(f):
if i >= main.settings_custom['auto_detection']['detection_settings']['number_lines']:
break
if re.search(re_tags_pos, line):
tagged_pos = True
break
f.seek(0)
for i, line in enumerate(f):
if i >= main.settings_custom['auto_detection']['detection_settings']['number_lines']:
break
if re.search(re_tags_non_pos, line):
tagged_non_pos = True
break
if tagged_pos and tagged_non_pos:
text_type = ('tokenized', 'tagged_both')
elif tagged_pos:
text_type = ('tokenized', 'tagged_pos')
elif tagged_non_pos:
text_type = ('untokenized', 'tagged_non_pos')
else:
text_type = ('untokenized', 'untagged')
success = True
except:
text_type = main.settings_custom['auto_detection']['default_settings']['default_text_type']
success = False
return text_type, success
def __init__(self, main, file, flat_tokens = True):
self.main = main
self.lang = file['lang']
self.text_type = file['text_type']
self.offsets_paras = []
self.offsets_sentences = []
self.offsets_clauses = []
if flat_tokens:
self.tokens_hierarchical = [[[[]]]]
else:
self.tokens_hierarchical = []
self.tokens_flat = []
self.tags_all = []
self.tags_pos = []
self.tags_non_pos = []
re_tags_all = wordless_matching.get_re_tags(main, tags = 'all')
re_tags_pos = wordless_matching.get_re_tags(main, tags = 'pos')
re_tags_non_pos = wordless_matching.get_re_tags(main, tags = 'non_pos')
with open(file['path'], 'r', encoding = file['encoding']) as f:
# Untokenized / Untagged
if self.text_type == ('untokenized', 'untagged'):
if flat_tokens:
for line in f:
text = line.rstrip()
if text:
tokens = wordless_text_processing.wordless_word_tokenize(main, text,
lang = self.lang,
flat_tokens = True)
self.tokens_hierarchical[0][0][0].extend(tokens)
else:
for line in f:
text = line.rstrip()
if text:
tokens = wordless_text_processing.wordless_word_tokenize(main, text,
lang = self.lang,
flat_tokens = False)
self.tokens_hierarchical.append(tokens)
# Untokenized / Tagged (Non-POS)
elif self.text_type == ('untokenized', 'tagged_non_pos'):
if flat_tokens:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
tokens = wordless_text_processing.wordless_word_tokenize(main, text_no_tags,
lang = self.lang,
flat_tokens = True)
self.tokens_hierarchical[0][0][0].extend(tokens)
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_non_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_non_pos.append([])
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
self.tokenize_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.tokenize_text(text)
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
tokens = wordless_text_processing.wordless_word_tokenize(main, text_no_tags,
lang = self.lang,
flat_tokens = False)
self.tokens_hierarchical.append(tokens)
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_non_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_non_pos.append([])
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
self.tokenize_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.tokenize_text(text)
# Tokenized / Untagged
elif self.text_type == ('tokenized', 'untagged'):
if flat_tokens:
for line in f:
text = line.rstrip()
if text:
self.tokens_hierarchical[0][0][0].extend(text.split())
else:
for line in f:
text = line.rstrip()
if text:
self.tokens_hierarchical.append([])
for sentence in wordless_text_processing.wordless_sentence_tokenize(main, text,
lang = self.lang):
self.tokens_hierarchical[-1].append([])
for clause in wordless_text_processing.wordless_clause_tokenize(main, sentence,
lang = self.lang):
self.tokens_hierarchical[-1][-1].append(clause.split())
# Tokenized / Tagged (POS)
elif self.text_type == ('tokenized', 'tagged_pos'):
if flat_tokens:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_hierarchical[0][0][0].extend(text_no_tags.split())
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_pos.append([])
# Extract tags
for tag in re.findall(re_tags_pos, text):
i_tag = text.index(tag)
self.split_text(text[:i_tag])
self.tags_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_hierarchical.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
for sentence in wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang):
self.tokens_hierarchical[-1].append([])
for clause in wordless_text_processing.wordless_clause_tokenize(main, sentence,
lang = self.lang):
self.tokens_hierarchical[-1][-1].append(clause.split())
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_pos.append([])
# Extract tags
for tag in re.findall(re_tags_pos, text):
i_tag = text.index(tag)
self.split_text(text[:i_tag])
self.tags_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
# Tokenized / Tagged (Non-POS)
elif self.text_type == ('tokenized', 'tagged_non_pos'):
if flat_tokens:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_hierarchical[0][0][0].extend(text_no_tags.split())
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_non_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_non_pos.append([])
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
self.split_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_hierarchical.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
for sentence in wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang):
self.tokens_hierarchical[-1].append([])
for clause in wordless_text_processing.wordless_clause_tokenize(main, sentence,
lang = self.lang):
self.tokens_hierarchical[-1][-1].append(clause.split())
# Check if the first token in the text is a tag
if i == 0 and re.match(re_tags_non_pos, text):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_non_pos.append([])
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
self.split_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
# Tokenized / Tagged (Both)
elif self.text_type == ('tokenized', 'tagged_both'):
if flat_tokens:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_all, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_hierarchical[0][0][0].extend(text_no_tags.split())
# Check if the first token in the text is a tag
if i == 0 and (re.match(re_tags_pos, text) or re.match(re_tags_non_pos, text)):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_all.append([])
self.tags_pos.append([])
self.tags_non_pos.append([])
# Extract tags
while text:
tag_pos = re.search(re_tags_pos, text)
tag_non_pos = re.search(re_tags_non_pos, text)
if tag_pos:
i_tag_pos = text.index(tag_pos.group())
if tag_non_pos:
i_tag_non_pos = text.index(tag_non_pos.group())
if (tag_pos and tag_non_pos and i_tag_pos < i_tag_non_pos or
tag_pos and not tag_non_pos):
self.split_text(text[:i_tag_pos])
self.tags_pos[-1].append(tag_pos.group())
self.tags_all[-1].append(tag_pos.group())
text = text[i_tag_pos + len(tag_pos.group()):]
elif (tag_pos and tag_non_pos and i_tag_pos > i_tag_non_pos or
not tag_pos and tag_non_pos):
self.split_text(text[:i_tag_non_pos])
self.tags_all[-1].append(tag_non_pos.group())
self.tags_non_pos[-1].append(tag_non_pos.group())
text = text[i_tag_non_pos + len(tag_non_pos.group()):]
else:
self.split_text(text)
break
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_hierarchical.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_all, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
for sentence in wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang):
self.tokens_hierarchical[-1].append([])
for clause in wordless_text_processing.wordless_clause_tokenize(main, sentence,
lang = self.lang):
self.tokens_hierarchical[-1][-1].append(clause.split())
# Check if the first token in the text is a tag
if i == 0 and (re.match(re_tags_pos, text) or re.match(re_tags_non_pos, text)):
self.tokens_hierarchical[0][0][0].insert(0, '')
self.tags_all.append([])
self.tags_pos.append([])
self.tags_non_pos.append([])
# Extract tags
while text:
tag_pos = re.search(re_tags_pos, text)
tag_non_pos = re.search(re_tags_non_pos, text)
if tag_pos:
i_tag_pos = text.index(tag_pos.group())
if tag_non_pos:
i_tag_non_pos = text.index(tag_non_pos.group())
if (tag_pos and tag_non_pos and i_tag_pos < i_tag_non_pos or
tag_pos and not tag_non_pos):
self.split_text(text[:i_tag_pos])
self.tags_all[-1].append(tag_pos.group())
self.tags_pos[-1].append(tag_pos.group())
text = text[i_tag_pos + len(tag_pos.group()):]
elif (tag_pos and tag_non_pos and i_tag_pos > i_tag_non_pos or
not tag_pos and tag_non_pos):
self.split_text(text[:i_tag_non_pos])
self.tags_all[-1].append(tag_non_pos.group())
self.tags_non_pos[-1].append(tag_non_pos.group())
text = text[i_tag_non_pos + len(tag_non_pos.group()):]
else:
self.split_text(text)
break
# Paragraph, sentence and clause offsets
for para in self.tokens_hierarchical:
self.offsets_paras.append(len(self.tokens_flat))
for sentence in para:
self.offsets_sentences.append(len(self.tokens_flat))
for clause in sentence:
self.offsets_clauses.append(len(self.tokens_flat))
self.tokens_flat.extend(clause)
# Tags
if self.text_type[1] == 'tagged_pos':
self.tags_non_pos = [[] for i in range(len(self.tokens_flat))]
self.tags_all = copy.deepcopy(self.tags_pos)
elif self.text_type[1] == 'tagged_non_pos':
self.tags_pos = [[] for i in range(len(self.tokens_flat))]
self.tags_all = copy.deepcopy(self.tags_non_pos)
elif self.text_type[1] == 'untagged':
self.tags_all = [[] for i in range(len(self.tokens_flat))]
self.tags_pos = [[] for i in range(len(self.tokens_flat))]
self.tags_non_pos = [[] for i in range(len(self.tokens_flat))]
# Remove whitespace around all tags
self.tags_all = [[tag.strip() for tag in tags] for tags in self.tags_all]
self.tags_pos = [[tag.strip() for tag in tags] for tags in self.tags_pos]
self.tags_non_pos = [[tag.strip() for tag in tags] for tags in self.tags_non_pos]
def wordless_lemmatize(main,
tokens,
lang,
text_type=('untokenized', 'untagged'),
lemmatizer='default'):
empty_offsets = []
mapping_lemmas = {}
lemmas = []
tokens = [str(token) for token in tokens]
re_tags_all = wordless_matching.get_re_tags(main, tags='all')
re_tags_pos = wordless_matching.get_re_tags(main, tags='pos')
re_tags_non_pos = wordless_matching.get_re_tags(main, tags='non_pos')
if text_type[1] == 'tagged_both':
tags = [''.join(re.findall(re_tags_all, token)) for token in tokens]
tokens = [re.sub(re_tags_all, '', token) for token in tokens]
elif text_type[1] == 'tagged_pos':
tags = [''.join(re.findall(re_tags_pos, token)) for token in tokens]
tokens = [re.sub(re_tags_pos, '', token) for token in tokens]
elif text_type[1] == 'tagged_non_pos':
tags = [
''.join(re.findall(re_tags_non_pos, token)) for token in tokens
]
tokens = [re.sub(re_tags_non_pos, '', token) for token in tokens]
else:
tags = [''] * len(tokens)
# Record empty tokens
for i, token in reversed(list(enumerate(tokens))):
if not token.strip():
tokens.remove(token)
empty_offsets.append(i)
wordless_text_utils.check_lemmatizers(main, lang)
if tokens and lang in main.settings_global['lemmatizers']:
if lemmatizer == 'default':
lemmatizer = main.settings_custom['lemmatization']['lemmatizers'][
lang]
# Dutch, English, French, German, Greek (Modern), Italian, Portuguese, Spanish
if 'spaCy' in lemmatizer:
nlp = main.__dict__[f'spacy_nlp_{lang}']
doc = spacy.tokens.Doc(nlp.vocab, words=tokens)
nlp.tagger(doc)
lemmas = [token.lemma_ for token in doc]
# English
elif lemmatizer == main.tr('NLTK - WordNet Lemmatizer'):
word_net_lemmatizer = nltk.WordNetLemmatizer()
for token, pos in wordless_pos_tag(
main,
tokens,
lang='eng',
pos_tagger='NLTK - Perceptron POS Tagger',
tagset='universal'):
if pos == 'ADJ':
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.ADJ))
elif pos in ['NOUN', 'PROPN']:
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.NOUN))
elif pos == 'ADV':
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.ADV))
elif pos in ['VERB', 'AUX']:
lemmas.append(
word_net_lemmatizer.lemmatize(
token, pos=nltk.corpus.wordnet.VERB))
else:
lemmas.append(word_net_lemmatizer.lemmatize(token))
# Greek (Ancient)
elif lemmatizer == main.tr(
'lemmalist-greek - Greek (Ancient) Lemma List'):
with open(wordless_misc.get_abs_path(
'lemmatization/lemmalist-greek/lemmalist-greek.txt'),
'r',
encoding='utf_8') as f:
for line in f.readlines():
line = line.rstrip()
if line:
lemma, *words = line.split()
for word in words:
mapping_lemmas[word] = lemma
# Russian & Ukrainian
elif lemmatizer == main.tr('pymorphy2 - Morphological Analyzer'):
if lang == 'rus':
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='ru')
else:
morphological_analyzer = pymorphy2.MorphAnalyzer(lang='uk')
for token in tokens:
lemmas.append(
morphological_analyzer.parse(token)[0].normal_form)
# Tibetan
elif lemmatizer == main.tr('pybo - Tibetan Lemmatizer'):
word_tokenizer = main.settings_custom['word_tokenization'][
'word_tokenizers'][lang]
wordless_text_utils.check_pybo_tokenizers(
main, word_tokenizer=word_tokenizer)
if word_tokenizer == main.tr(
'pybo - Tibetan Word Tokenizer (GMD)'):
tokens = main.pybo_tokenizer_gmd.tokenize(' '.join(tokens))
elif word_tokenizer == main.tr(
'pybo - Tibetan Word Tokenizer (POS)'):
tokens = main.pybo_tokenizer_pos.tokenize(' '.join(tokens))
elif word_tokenizer == main.tr(
'pybo - Tibetan Word Tokenizer (tsikchen)'):
tokens = main.pybo_tokenizer_tsikchen.tokenize(
' '.join(tokens))
for token in tokens:
if token.lemma:
lemmas.append(token.lemma)
else:
lemmas.append(token.text)
# Other Languages
elif 'Lemmatization Lists' in lemmatizer:
lang = wordless_conversion.to_iso_639_1(main, lang)
with open(wordless_misc.get_abs_path(
f'lemmatization/Lemmatization Lists/lemmatization-{lang}.txt'
),
'r',
encoding='utf_8_sig') as f:
for line in f:
try:
lemma, word = line.rstrip().split('\t')
mapping_lemmas[word] = lemma
except:
pass
else:
lemmas = tokens
if mapping_lemmas:
lemmas = [mapping_lemmas.get(token, token) for token in tokens]
# Insert empty lemmas
for empty_offset in empty_offsets:
lemmas.insert(empty_offset, '')
return [lemma + tag for lemma, tag in zip(lemmas, tags)]
def __init__(self, main, file, tokens_only = True):
self.main = main
self.lang = file['lang']
self.text_type = file['text_type']
self.para_offsets = []
self.sentence_offsets = []
self.tokens_sentences_paras = []
self.tokens = []
self.tags_all = []
self.tags_pos = []
self.tags_non_pos = []
re_tags_all = wordless_matching.get_re_tags(main, tags = 'all')
re_tags_pos = wordless_matching.get_re_tags(main, tags = 'pos')
re_tags_non_pos = wordless_matching.get_re_tags(main, tags = 'non_pos')
if tokens_only:
keep_sentences = False
else:
keep_sentences = True
with open(file['path'], 'r', encoding = file['encoding']) as f:
# Untokenized / Untagged
if self.text_type == ('untokenized', 'untagged'):
for line in f:
text = line.rstrip()
if text:
tokens_sentences = wordless_text_processing.wordless_word_tokenize(main, text,
lang = self.lang,
keep_sentences = keep_sentences)
self.tokens_sentences_paras.append(tokens_sentences)
# Untokenized / Tagged (Non-POS)
elif self.text_type == ('untokenized', 'tagged_non_pos'):
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
tokens_sentences = wordless_text_processing.wordless_word_tokenize(main, text_no_tags,
lang = self.lang,
keep_sentences = keep_sentences)
self.tokens_sentences_paras.append(tokens_sentences)
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
if i == 0 and i_tag == 0 and not self.tags_non_pos:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_non_pos.append([tag])
else:
self.tokenize_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.tokenize_text(text)
# Tokenized / Untagged
elif self.text_type == ('tokenized', 'untagged'):
if tokens_only:
for line in f:
text = line.rstrip()
if text:
self.tokens_sentences_paras.append([text.split()])
else:
for line in f:
text = line.rstrip()
if text:
self.tokens_sentences_paras.append([])
for sentence in wordless_text_processing.wordless_sentence_tokenize(main, text,
lang = self.lang):
self.tokens_sentences_paras[-1].append(sentence.split())
# Tokenized / Tagged (POS)
elif self.text_type == ('tokenized', 'tagged_pos'):
if tokens_only:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_sentences_paras.append([text_no_tags.split()])
# Extract tags
for tag in re.findall(re_tags_pos, text):
i_tag = text.index(tag)
if self.tokens_sentences_paras:
self.split_text(text[:i_tag])
self.tags_pos[-1].append(tag)
else:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_pos.append([tag])
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_sentences_paras.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
sentences = wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang)
for sentence in sentences:
self.tokens_sentences_paras[-1].append(sentence.split())
# Extract tags
for tag in re.findall(re_tags_pos, text):
i_tag = text.index(tag)
if self.tokens_sentences_paras:
self.split_text(text[:i_tag])
self.tags_pos[-1].append(tag)
else:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_pos.append([tag])
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
# Tokenized / Tagged (Non-POS)
elif self.text_type == ('tokenized', 'tagged_non_pos'):
if tokens_only:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_sentences_paras.append([text_no_tags.split()])
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
if i == 0 and i_tag == 0 and not self.tags_non_pos:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_non_pos.append([tag])
else:
self.split_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_sentences_paras.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_non_pos, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
sentences = wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang)
for sentence in sentences:
self.tokens_sentences_paras[-1].append(sentence.split())
# Extract tags
for tag in re.findall(re_tags_non_pos, text):
i_tag = text.index(tag)
if i == 0 and i_tag == 0 and not self.tags_non_pos:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_non_pos.append([tag])
else:
self.split_text(text[:i_tag])
self.tags_non_pos[-1].append(tag)
text = text[i_tag + len(tag):]
# The last part of the text
if text:
self.split_text(text)
# Tokenized / Tagged (Both)
elif self.text_type == ('tokenized', 'tagged_both'):
if tokens_only:
for i, line in enumerate(f):
text = line.rstrip()
if text:
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_all, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
self.tokens_sentences_paras.append([text_no_tags.split()])
# Extract tags
while text:
tag_pos = re.search(re_tags_pos, text)
tag_non_pos = re.search(re_tags_non_pos, text)
if tag_pos:
i_tag_pos = text.index(tag_pos.group())
if tag_non_pos:
i_tag_non_pos = text.index(tag_non_pos.group())
if (tag_pos and tag_non_pos and i_tag_pos < i_tag_non_pos or
tag_pos and not tag_non_pos):
if i == 0 and i_tag_pos == 0 and not self.tags_all:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_all.append([tag_pos.group()])
self.tags_pos.append([tag_pos.group()])
self.tags_non_pos.append([])
else:
self.split_text(text[:i_tag_pos])
self.tags_pos[-1].append(tag_pos.group())
self.tags_all[-1].append(tag_pos.group())
text = text[i_tag_pos + len(tag_pos.group()):]
elif (tag_pos and tag_non_pos and i_tag_pos > i_tag_non_pos or
not tag_pos and tag_non_pos):
if i == 0 and i_tag_non_pos == 0 and not self.tags_all:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_all.append([tag_non_pos.group()])
self.tags_pos.append([])
self.tags_non_pos.append([tag_non_pos.group()])
else:
self.split_text(text[:i_tag_non_pos])
self.tags_all[-1].append(tag_non_pos.group())
self.tags_non_pos[-1].append(tag_non_pos.group())
text = text[i_tag_non_pos + len(tag_non_pos.group()):]
else:
self.split_text(text)
break
else:
for i, line in enumerate(f):
text = line.rstrip()
if text:
self.tokens_sentences_paras.append([])
# Replace all tags with a whitespace to ensure no words run together
text_no_tags = re.sub(re_tags_all, ' ', text)
text_no_tags = re.sub(r'\s+', ' ', text_no_tags)
sentences = wordless_text_processing.wordless_sentence_tokenize(main, text_no_tags,
lang = self.lang)
for sentence in sentences:
self.tokens_sentences_paras[-1].append(sentence.split())
# Extract tags
while text:
tag_pos = re.search(re_tags_pos, text)
tag_non_pos = re.search(re_tags_non_pos, text)
if tag_pos:
i_tag_pos = text.index(tag_pos.group())
if tag_non_pos:
i_tag_non_pos = text.index(tag_non_pos.group())
if (tag_pos and tag_non_pos and i_tag_pos < i_tag_non_pos or
tag_pos and not tag_non_pos):
if i == 0 and i_tag_pos == 0 and not self.tags_all:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_all.append([tag_pos.group()])
self.tags_pos.append([tag_pos.group()])
self.tags_non_pos.append([])
else:
self.split_text(text[:i_tag_pos])
self.tags_all[-1].append(tag_pos.group())
self.tags_pos[-1].append(tag_pos.group())
text = text[i_tag_pos + len(tag_pos.group()):]
elif (tag_pos and tag_non_pos and i_tag_pos > i_tag_non_pos or
not tag_pos and tag_non_pos):
if i == 0 and i_tag_non_pos == 0 and not self.tags_all:
self.tokens_sentences_paras[0][0].insert(0, '')
self.tags_all.append([tag_non_pos.group()])
self.tags_pos.append([])
self.tags_non_pos.append([tag_non_pos.group()])
else:
self.split_text(text[:i_tag_non_pos])
self.tags_all[-1].append(tag_non_pos.group())
self.tags_non_pos[-1].append(tag_non_pos.group())
text = text[i_tag_non_pos + len(tag_non_pos.group()):]
else:
self.split_text(text)
break
# Record paragraph and sentence offsets
for tokens_sentences in self.tokens_sentences_paras:
self.para_offsets.append(len(self.tokens))
for tokens in tokens_sentences:
self.sentence_offsets.append(len(self.tokens))
self.tokens.extend(tokens)
if self.text_type[1] == 'tagged_pos':
self.tags_non_pos = [[] for i in range(len(self.tokens))]
self.tags_all = copy.deepcopy(self.tags_pos)
elif self.text_type[1] == 'tagged_non_pos':
self.tags_pos = [[] for i in range(len(self.tokens))]
self.tags_all = copy.deepcopy(self.tags_non_pos)
elif self.text_type[1] == 'untagged':
self.tags_all = [[] for i in range(len(self.tokens))]
self.tags_pos = [[] for i in range(len(self.tokens))]
self.tags_non_pos = [[] for i in range(len(self.tokens))]
# Remove whitespace around all tags
self.tags_all = [[tag.strip() for tag in tags] for tags in self.tags_all]
self.tags_pos = [[tag.strip() for tag in tags] for tags in self.tags_pos]
self.tags_non_pos = [[tag.strip() for tag in tags] for tags in self.tags_non_pos]