def analyse_words(vocab):
analysis_types = set()
problem_words = set()
word_analyses = dict()
for word in vocab:
e_output = estnltk.Text(word, **estnltk_options)
# mostly accents within a word, causing estnltk to detect multiple words
if len(e_output.analysis) > 1:
problem_words.add(word)
continue
elif len(e_output.analysis) == 0:
raise Exception(
"no analysis returned from the analyzer for word: %s" % word)
word_analyses[word] = list()
for e_analysis in e_output.analysis[0]:
analysis_string = get_analysis_string(e_analysis)
analysis_types.add(analysis_string)
if analysis_string not in word_analyses[word]:
word_analyses[word].append(analysis_string)
e_cap_output = estnltk.Text(word.capitalize(), **estnltk_options)
if len(e_cap_output.analysis) != 1:
raise Exception("problem in capital word analysis for word %s" %
word)
for e_analysis in e_cap_output.analysis[0]:
analysis_string = get_analysis_string(e_analysis)
analysis_types.add(analysis_string)
if analysis_string not in word_analyses[word]:
word_analyses[word].append(analysis_string)
return word_analyses, analysis_types, problem_words
def process_frequency_list():
freq_dict = {}
print("Töötlen sagedusloendit")
with open("freq_list_raw.txt", "r", encoding="UTF-8-SIG") as file:
count = 0
for line in file:
if "?" in line:
continue
count += 1
print(count)
parts = line.strip().split(" ")
freq = int(parts[0])
lemma = parts[1]
word_analysis = estnltk.Text(lemma).tag_analysis()
synsets = wn.synsets(lemma)
if is_tag_allowed(word_analysis) and len(
synsets) > 0 and has_relations(synsets):
freq_dict[lemma] = freq
print("Koostan töödeldud sagedusloendit")
with open("freq_list_processed.txt", "w", encoding="UTF-8") as file:
for word in sorted(freq_dict, key=freq_dict.get, reverse=True):
file.write(word + " " + str(freq_dict[word]) + "\n")
freq_dict.clear()
print("Sagedusloend töödeldud")
def analyze_title(title):
tc = 0
article["title"] = title
article["scount"] += 1
# eemalda märgendid
detagged_title = re.sub("<.*?>", "", title)
# jaga sõnadeks
title_words = estnltk.Text(
re.sub("[\s,.!?;:\xAB\xBB\-\"()„’́'“«»]+", " ", detagged_title))
title_lemmas = title_words.lemmas
for lemma in title_lemmas:
if len(lemma) > 0:
if lemma.lower() not in words:
words[lemma.lower()] = 0
words[lemma.lower()] += 5
tc += 1
article["tlength"] = tc
article["wcount"] += tc
def word_based_score():
norm_word_weights()
for elem in article["body"]:
content = elem["content"]
# eemaldame märgendid
content = re.sub("<.*?>", "", content)
# jagame sõnadeks
content_words = estnltk.Text(
re.sub("[\s,.!?;:\xAB\xBB\-\"()„’́'“«»]+", " ", content))
# lemmastame
content_lemmas = content_words.lemmas
for lemma in content_lemmas:
lemma = lemma.lower()
if lemma in words:
elem["wrdsc"] += words[lemma]
if elem["wcount"] != 0:
elem["wrdsc"] /= elem["wcount"]
else:
elem["wrdsc"] = 0
# normaliseerime lausete sõnasageduse põhise skoori
normalize_score("wrdsc")
def avg_char_count_in_sentence(text) -> float:
words = (estnltk.Text(text)).tokenize_words().words
count_of_words = len(words)
count_of_chars = 0
for w in words:
single_word = str(w["text"])
if single_word.isalpha():
count_of_chars_in_word = len(list(single_word))
count_of_chars += count_of_chars_in_word
return count_of_chars / count_of_words
def order_by_similarity(processed_list, original_word_lemma):
if len(processed_list) < 2:
return processed_list
temp_dict = {}
for temp_word in processed_list:
temp_dict[temp_word] = get_word_similarity(
original_word_lemma,
estnltk.Text(temp_word).lemmas[0])
return sorted(temp_dict, key=temp_dict.get, reverse=True)
def getParahgraphsDetails(inputText):
document = estnltk.Text(inputText)
total_parahs = len(document.paragraph_texts)
lengths = []
for parah in document.paragraph_texts:
words = text.text_to_word_sequence(
parah,
filters=' !#$%&()*+,-./:;<=>?@[\\]^_{|}~\t\n"',
lower=True,
split=" ")
lengths.append(len(words))
lengths = np.array(lengths)
average_length = lengths.mean()
return [total_parahs, average_length]
def process_foreign_list():
basic = dictionaries.get_basic_list("basic_processed.txt")
foreign_words = {}
count = 0
print("Töötlen selgitustega võõrsõnade loendit")
with open("foreign_meaning.txt", "r", encoding="UTF-8") as file:
for line in file:
print(count)
count += 1
parts = line.strip().split("\t")
word = parts[0]
if len(parts) > 1:
definition = parts[1].split("(")[0]
foreign_words[word] = definition
else:
foreign_words[word] = None
print("Töötlen märksõnadega võõrsõnade loendit")
with open("foreign_keywords.txt", "r", encoding="UTF-8") as file:
count = 0
for word in file:
print(count)
count += 1
word = word.strip()
if word[0] == "-" or word[-1] == "-":
continue
word_analysis = estnltk.Text(word).tag_analysis()
synsets = wn.synsets(word)
# filtreerime välja kõik ebavajalikud sõnad
if is_tag_allowed(
word_analysis
) and word not in basic and word not in foreign_words and has_relations(
synsets):
foreign_words[word] = None
print("Koostan töödeldud võõrsõnade loendit")
with open("foreign_processed.txt", "w", encoding="UTF-8") as file:
for word in foreign_words:
if len(word.split(" ")) <= 1:
if foreign_words[word] is None:
file.write(word + "\n")
else:
file.write(word + " " + foreign_words[word] + "\n")
foreign_words.clear()
print("Võõrsõnade loend töödeldud")
def get_most_similar(word_list, original_lemma):
if len(word_list) == 1:
return word_list[0]
# word2vec ei arvesta käändes olevaid sõnu, seega ei ole see optimaalne
best_similarity = -1
best_lemma = None
for word in word_list:
word_analysis = estnltk.Text(word).analysis[0][0]
similarity = get_word_similarity(original_lemma,
word_analysis[LEMMA],
default=0)
if similarity > best_similarity:
best_lemma = word
best_similarity = similarity
return best_lemma
def process_basic_list():
basic_list = []
print("Töötlen põhisõnavara loendit")
count = 0
with open("basic_raw.txt", "r", encoding="UTF-8") as file:
for line in file:
print(count)
count += 1
word = line.strip()
word_analysis = estnltk.Text(word).tag_analysis()
synsets = wn.synsets(word)
if is_tag_allowed(word_analysis) and len(
synsets) > 0 and has_relations(synsets):
basic_list.append(word)
print("Koostan töödeldud põhisõnavara loendit")
with open("basic_processed.txt", "w", encoding="UTF-8") as file:
for word in basic_list:
file.write(word + "\n")
basic_list.clear()
print("Põhisõnavara loend töödeldud")
def analyze_line(line):
global sentence
global parnr
global divnr
if bool(re.search("<p>", line)):
article["pcount"] += 1
parnr += 1
return
if bool(re.search('</p>', line)) or bool(re.search("</div", line)):
return
# lause informatsioon
sentence = {
"wcount": 0,
"parnr": 0,
"divnr": 0,
"subhead": 0,
"bibl": 0,
"caption": 0,
"possc": 0,
"forsc": 0,
"wrdsc": 0,
"score": 0,
"content": ""
}
if bool(re.search("<div", line)):
if len(article["body"]) != 0:
article["divcount"] += 1
divnr += 1
sentence["subhead"] = 1
article["pcount"] += 1
parnr += 1
# eemaldame märgendused
line = re.sub("<.*?>", "", line)
if bool(re.search("<bibl>", line)):
sentence["bibl"] = 1
if bool(re.search("Pildi allkiri", line)):
sentence["caption"] = 1
sentence["content"] = line
sentence["parnr"] = parnr
sentence["divnr"] = divnr
# eemaldame märgendid
line = re.sub("<.*?>", "", line)
# jagame sõnadeks
line_words = estnltk.Text(
re.sub("[\s,.!?;:\xAB\xBB\-\"()„’́'“«»]+", " ", line))
line_lemmas = line_words.lemmas
sentence["wcount"] += len(line_lemmas)
weight = 1
if sentence["subhead"] > 0:
weight += 1
for lemma in line_lemmas:
lemma = lemma.lower()
if lemma.isalpha():
if lemma not in words:
words[lemma] = 0
words[lemma] += weight
body = article["body"]
body.append(sentence)
article["body"] = body
article["wcount"] += sentence["wcount"]